test: add comprehensive unit, integration, and e2e test suites

Introduce complete test coverage for the ARTEMIS platform across multiple layers:
Unit tests: Add tests for agent nodes (Planner, Operator, Validator, Diagnoser, FlashRunner, Explorer), tools, utils (OCR/XML fusion, UI hierarchy), and data engine.
Integration tests: Add live tool verification, reactive runner, IME, and standalone workflow tests.
E2E tests: Add end-to-end scenario tests for media playback and failure recovery flows.
Fixtures & mocks: Add mock inputs (audio, video, screenshot) and test utilities.
This commit is contained in:
chiwanko11 committed 2026-08-13 12:18:32 -07:00
1 parent c1803fcb86
commit 5d2283deaa
115 files changed
+22813

No files matched your search

@@ -0,0 +1,68 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Helper utilities for Explorer integration tests."""
from pathlib import Path
from PIL import Image
from unittest.mock import MagicMock
from artemis.context import ArtemisContext
from artemis.graph.state import State
def create_mock_context() -> MagicMock:
"""Creates a mock ArtemisContext suitable for Explorer tests."""
ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.llm_config = MagicMock()
ctx.agent_config = MagicMock()
ctx.agent_config.blacklisted_tools = {"explorer": []}
ctx.data_engine = None
ctx.adb_client = None
return ctx
def create_mock_state() -> MagicMock:
"""Creates a mock State suitable for Explorer tests."""
state = MagicMock(spec=State)
state.operator_raw_data = {"width": 1080, "height": 2400}
state.latest_ui_hierarchy = []
state.latest_screenshot = None
return state
def get_or_create_test_screenshot(preferred_path: Path | None = None) -> Path:
"""Finds an existing test screenshot or generates a valid 1080x2400 mock JPEG."""
if preferred_path and preferred_path.exists() and preferred_path.stat().st_size > 0:
return preferred_path
# Search known repo screenshot paths
candidate_paths = [
Path(__file__).parent / "input_screenshot_test_explorer_all_tools_sequential_mocked.jpg",
Path(__file__).parent / "input_screenshot_test_explorer_ask_vision_coder_tool.jpg",
Path(__file__).parents[3] / "tools" / "inputs" / "screenshot.jpg",
]
for p in candidate_paths:
if p.exists() and p.stat().st_size > 0:
return p
# If none found, generate a mock RGB image
out_path = (
Path(__file__).parent / "input_screenshot_test_explorer_all_tools_sequential_mocked.jpg"
)
img = Image.new("RGB", (1080, 2400), color=(73, 109, 137))
img.save(out_path, format="JPEG")
return out_path
@@ -0,0 +1,280 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import json
import logging
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.agents.explorer.explorer import Explorer
from tests.integration.agents.explorer.test_explorer_all_tools.helpers import (
create_mock_context,
create_mock_state,
)
from tests.integration.agents.explorer.test_explorer_all_tools.seed_mock_image import (
seed_database,
)
# Set up logging to stdout
logging.basicConfig(level=logging.INFO, format="%(message)s")
logger = logging.getLogger("live_tools_test")
def extract_and_print_tool_results(contents, set_idx, logger):
logger.info("\n======================================================================")
logger.info(f" RESULTS FOR INPUT TEST SET {set_idx}")
logger.info("======================================================================")
for content in contents:
if content.role != "tool":
continue
for part in content.parts:
if part.function_response:
name = part.function_response.name
response_data = part.function_response.response
# Extract result text
tool_text = None
if hasattr(response_data, "get"):
tool_text = response_data.get("result")
elif hasattr(response_data, "fields"):
tool_text = response_data.fields.get("result")
else:
tool_text = getattr(response_data, "result", None)
logger.info(f"\n🛠️ Tool: '{name}'")
logger.info("--------------------------------------------------")
logger.info(f"Output Result:\n{tool_text}")
if part.inline_data and part.inline_data.mime_type == "image/jpeg":
logger.info(
f"📷 Visual Data: Attached {len(part.inline_data.data)}"
" bytes of annotated JPEG screenshot"
)
async def run_single_set(set_idx, query, search_query, labels, coords_nx, coords_ny, candidates):
import os
os.environ["GOOGLE_API_KEY"] = "dummy_api_key"
mock_ctx = create_mock_context()
mock_ctx.llm_config.utils = MagicMock()
mock_ctx.llm_config.utils.object_detector = MagicMock()
mock_ctx.llm_config.utils.object_detector.model = "gemini-2.5-flash"
mock_ctx.llm_config.utils.object_detector.timeout = 10.0
mock_state = create_mock_state()
screenshot_path = (
Path(__file__).resolve().parent
/ "input_screenshot_test_explorer_all_tools_sequential_mocked.jpg"
)
# Multi-turn sequential mock client
# Turn 1: get_ocr_list
mock_fc_ocr = MagicMock()
mock_fc_ocr.name = "get_ocr_list"
mock_fc_ocr.args = {}
mock_resp_ocr = MagicMock()
mock_resp_ocr.function_calls = [mock_fc_ocr]
# Turn 2: search_xml_ocr with query
mock_fc_search = MagicMock()
mock_fc_search.name = "search_xml_ocr"
mock_fc_search.args = {"query": search_query}
mock_resp_search = MagicMock()
mock_resp_search.function_calls = [mock_fc_search]
# Turn 3: detect_objects
mock_fc_detect = MagicMock()
mock_fc_detect.name = "detect_objects"
mock_fc_detect.args = {"labels": labels, "target_image_id": "img_0"}
mock_resp_detect = MagicMock()
mock_resp_detect.function_calls = [mock_fc_detect]
# Turn 4: search_xml_ocr with coordinates
mock_fc_coords = MagicMock()
mock_fc_coords.name = "search_xml_ocr"
mock_fc_coords.args = {"nx": coords_nx, "ny": coords_ny}
mock_resp_coords = MagicMock()
mock_resp_coords.function_calls = [mock_fc_coords]
# Turn 5: submit_answer
mock_fc_submit = MagicMock()
mock_fc_submit.name = "submit_answer"
mock_fc_submit.args = {
"candidates": candidates,
"fallback_message": f"Successfully verified test set {set_idx}",
}
mock_resp_submit = MagicMock()
mock_resp_submit.function_calls = [mock_fc_submit]
mock_client = MagicMock()
# Mock client.aio.models.generate_content side effects for ReAct turns
mock_client.aio.models.generate_content = AsyncMock(
side_effect=[
mock_resp_ocr,
mock_resp_search,
mock_resp_detect,
mock_resp_coords,
mock_resp_submit,
]
)
# Mock the internal model call for _detect_single_label inside detect_objects
mock_detect_response = MagicMock()
y_norm, x_norm = 0, 0
if "gear icon" in labels:
y_norm, x_norm = 300, 200
elif "dashboard icon" in labels:
y_norm, x_norm = 500, 500
elif "magnifying glass icon" in labels:
y_norm, x_norm = 140, 875
elif "avatar icon" in labels:
y_norm, x_norm = 825, 375
elif "arrow back icon" in labels:
y_norm, x_norm = 75, 100
mock_detect_response.text = json.dumps([{"point": [y_norm, x_norm]}])
mock_detector_client = MagicMock()
mock_detector_client.aio.models.generate_content = AsyncMock(return_value=mock_detect_response)
with (
patch(
"google.genai.Client",
side_effect=[mock_client, mock_detector_client],
),
):
explorer = Explorer(mock_ctx)
await explorer.run(
query=query,
context_feedback="",
screenshot_path=str(screenshot_path),
state=mock_state,
)
call_args_list = mock_client.aio.models.generate_content.call_args_list
contents = call_args_list[4].kwargs.get("contents") or call_args_list[4].args[0]
extract_and_print_tool_results(contents, set_idx, logger)
async def main():
import shutil
from artemis.config import settings
# Define outputs directory
outputs_dir = Path(__file__).resolve().parent / "outputs"
if outputs_dir.exists():
shutil.rmtree(outputs_dir)
outputs_dir.mkdir(parents=True, exist_ok=True)
# Redirect traces output to outputs/ and only keep the latest run
settings.TRACES_PATH = outputs_dir
# 1. Seed the SQLite DB record first
seed_database()
# 2. Define the 5 groups of input tests
test_sets = [
{
"query": "Verify Settings button",
"search_query": "Settings",
"labels": ["gear icon"],
"coords_nx": (185), # Matches pixel center [200, 225] on [1080, 2400] scale
"coords_ny": 93,
"candidates": [
{
"label": "S1",
"coords": [185, 93],
"description": "Settings Button",
}
],
},
{
"query": "Verify Dashboard button",
"search_query": "Dashboard",
"labels": ["dashboard icon"],
"coords_nx": (462), # Matches pixel center [500, 500] on [1080, 2400] scale
"coords_ny": 208,
"candidates": [
{
"label": "S1",
"coords": [462, 208],
"description": "Dashboard Button",
}
],
},
{
"query": "Verify Search bar",
"search_query": "Search",
"labels": ["magnifying glass icon"],
"coords_nx": (810), # Matches pixel center [875, 140] on [1080, 2400] scale
"coords_ny": 58,
"candidates": [
{
"label": "S1",
"coords": [810, 58],
"description": "Search bar",
}
],
},
{
"query": "Verify Profile icon",
"search_query": "Profile",
"labels": ["avatar icon"],
"coords_nx": (347), # Matches pixel center [375, 825] on [1080, 2400] scale
"coords_ny": 343,
"candidates": [
{
"label": "S1",
"coords": [347, 343],
"description": "Profile icon",
}
],
},
{
"query": "Verify Back button",
"search_query": "Back",
"labels": ["arrow back icon"],
"coords_nx": (92), # Matches pixel center [100, 75] on [1080, 2400] scale
"coords_ny": 31,
"candidates": [
{
"label": "S1",
"coords": [92, 31],
"description": "Back button",
}
],
},
]
# 3. Run all 5 sets sequentially
for i, test in enumerate(test_sets, 1):
await run_single_set(
set_idx=i,
query=test["query"],
search_query=test["search_query"],
labels=test["labels"],
coords_nx=test["coords_nx"],
coords_ny=test["coords_ny"],
candidates=test["candidates"],
)
if __name__ == "__main__":
asyncio.run(main())
@@ -0,0 +1,162 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import hashlib
import json
from pathlib import Path
import sqlite3
def seed_database():
db_path = Path(__file__).resolve().parents[5] / "traces" / "data_engine.db"
screenshot_path = (
Path(__file__).resolve().parent
/ "input_screenshot_test_explorer_all_tools_sequential_mocked.jpg"
)
if not screenshot_path.exists():
# Try fallback to tests/screenshots/pixel_phone_mockup.png
source_path = (
Path(__file__).resolve().parents[5] / "tests" / "screenshots" / "pixel_phone_mockup.png"
)
import shutil
shutil.copy(source_path, screenshot_path)
print(f"Copied screenshot from {source_path} to {screenshot_path}")
# Compute SHA-256 hash
sha256_hash = hashlib.sha256()
with open(screenshot_path, "rb") as f:
for byte_block in iter(lambda: f.read(4096), b""):
sha256_hash.update(byte_block)
computed_hash = sha256_hash.hexdigest()
print(f"Computed screenshot hash: {computed_hash}")
# Bounding elements
# 1. Settings [100, 200][300, 250] -> center pixel [200, 225] -> normalized [185, 93]
# 2. Dashboard [400, 400][600, 600] -> center pixel [500, 500] -> normalized [462, 208]
# 3. Search [800, 100][950, 180] -> center pixel [875, 140] -> normalized [810, 58]
# 4. Profile [300, 800][450, 850] -> center pixel [375, 825] -> normalized [347, 343]
# 5. Back [50, 50][150, 100] -> center pixel [100, 75] -> normalized [92, 31]
ocr_result = [
{"text": "Screen Dashboard Title", "position": []},
{
"text": "Settings",
"position": [
{"x": 100, "y": 200},
{"x": 300, "y": 200},
{"x": 300, "y": 250},
{"x": 100, "y": 250},
],
},
{
"text": "Dashboard",
"position": [
{"x": 400, "y": 400},
{"x": 600, "y": 400},
{"x": 600, "y": 600},
{"x": 400, "y": 600},
],
},
{
"text": "Search",
"position": [
{"x": 800, "y": 100},
{"x": 950, "y": 100},
{"x": 950, "y": 180},
{"x": 800, "y": 180},
],
},
{
"text": "Profile",
"position": [
{"x": 300, "y": 800},
{"x": 450, "y": 800},
{"x": 450, "y": 850},
{"x": 300, "y": 850},
],
},
{
"text": "Back",
"position": [
{"x": 50, "y": 50},
{"x": 150, "y": 50},
{"x": 150, "y": 100},
{"x": 50, "y": 100},
],
},
]
ui_tree = [
{"class": "android.widget.FrameLayout", "bounds": "[0,0][1080,2400]"},
{
"class": "android.widget.Button",
"text": "Settings",
"resource-id": "com.example.app:id/settings",
"clickable": "true",
"bounds": "[100,200][300,250]",
},
{
"class": "android.widget.Button",
"text": "Dashboard",
"resource-id": "com.example.app:id/dashboard",
"clickable": "true",
"bounds": "[400,400][600,600]",
},
{
"class": "android.widget.Button",
"text": "Search",
"resource-id": "com.example.app:id/search",
"clickable": "true",
"bounds": "[800,100][950,180]",
},
{
"class": "android.widget.Button",
"text": "Profile",
"resource-id": "com.example.app:id/profile",
"clickable": "true",
"bounds": "[300,800][450,850]",
},
{
"class": "android.widget.Button",
"text": "Back",
"resource-id": "com.example.app:id/back",
"clickable": "true",
"bounds": "[50,50][150,100]",
},
]
conn = sqlite3.connect(db_path)
cursor = conn.cursor()
cursor.execute(
"INSERT OR REPLACE INTO images (image_name, timestamp, ocr_result,"
" ui_tree, extra_metadata) VALUES (?, ?, ?, ?, ?)",
(
computed_hash,
1717398000.0,
json.dumps(ocr_result),
json.dumps(ui_tree),
json.dumps({}),
),
)
conn.commit()
conn.close()
print(f"Successfully inserted mock image record {computed_hash} into database at {db_path}")
if __name__ == "__main__":
seed_database()
@@ -0,0 +1,62 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, patch
from artemis.agents.object_detector.object_detector import _run_object_detection
import pytest
@pytest.mark.asyncio
async def test_coordinate_normalization_swap():
"""Verify that _run_object_detection correctly swaps coordinates from [y, x] to [x, y]."""
# 1. Define the mock output returned by the VLM task (VLM native format: [y_norm, x_norm])
mock_vlm_output = [
{
"label": "settings icon",
"point": [800, 200], # [y, x] -> y = 800, x = 200
}
]
# 2. Patch dependencies to run in a pure offline sandbox
with (
patch(
"artemis.agents.object_detector.object_detector._detect_single_label",
new_callable=AsyncMock,
return_value=mock_vlm_output,
),
patch("artemis.agents.object_detector.object_detector.get_llm"),
):
# Mock context setup
mock_ctx = AsyncMock()
mock_ctx.llm_config.utils.object_detector = None
# 3. Execute the function under test
json_output = await _run_object_detection(
ctx=mock_ctx,
image_bytes=b"fake_image_bytes",
queries=["settings icon"],
templates=["find the {label}"],
)
# 4. Verify that the coordinates were successfully swapped
result = json_output
detected_items = result["detected"]
assert len(detected_items) == 1
assert detected_items[0]["label"] == "settings icon"
# CRITICAL ASSERTION:
# The coordinate MUST be swapped from [800, 200] (y, x) to [200, 800] (x, y)
assert detected_items[0]["point"] == [200, 800]
@@ -0,0 +1,334 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import json
import logging
from pathlib import Path
import shutil
import sys
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.agents.explorer.explorer import Explorer
from tests.integration.agents.explorer.test_explorer_all_tools.helpers import (
create_mock_context,
create_mock_state,
get_or_create_test_screenshot,
)
import pytest
def _verify_all_tools_responses(contents: list, logger: logging.Logger):
"""Parses the multi-turn conversational history payload and maps tool outputs."""
logger.info("=== VERIFYING TOOL RESPONSES IN CONVERSATION HISTORY ===")
tool_responses = {}
for content in contents:
for part in content.parts:
if part.function_response:
name = part.function_response.name
response_data = part.function_response.response
# Extract result text
tool_text = None
if hasattr(response_data, "get"):
tool_text = response_data.get("result")
elif hasattr(response_data, "fields"):
tool_text = response_data.fields.get("result")
else:
tool_text = getattr(response_data, "result", None)
if name not in tool_responses:
tool_responses[name] = []
tool_responses[name].append({"text": tool_text, "has_image": False})
logger.info(f"Found tool response for '{name}':\n{tool_text}")
if part.inline_data and part.inline_data.mime_type == "image/jpeg":
# The image byte block corresponds to the preceding function response
if tool_responses:
last_tool = list(tool_responses.keys())[-1]
tool_responses[last_tool][-1]["has_image"] = True
tool_responses[last_tool][-1]["img_bytes"] = part.inline_data.data
logger.info(
f" -> Attached {len(part.inline_data.data)} bytes of"
f" JPEG inline visual content to '{last_tool}'"
)
return tool_responses
@pytest.mark.asyncio
async def test_explorer_all_tools_sequential_mocked():
"""Verify that the Explorer ReAct loop correctly sequences, dispatches, processes, and
annotates screenshot outputs for all available visual tools:
1. get_ocr_list
2. search_xml_ocr (query)
3. detect_objects
4. search_xml_ocr (coordinates)
5. submit_answer
"""
test_name = sys._getframe().f_code.co_name
# Set up standard log file saved on the same level as test file
log_file = Path(__file__).parent / f"log_{test_name}.log"
logger = logging.getLogger(test_name)
logger.setLevel(logging.INFO)
handler = logging.FileHandler(log_file, mode="w", encoding="utf-8")
logger.addHandler(handler)
# Attach the main agent logger to capture internal step logs
agent_logger = logging.getLogger("artemis.agents.explorer.explorer")
agent_logger.setLevel(logging.INFO)
agent_logger.addHandler(handler)
logger.info(f"Starting comprehensive test: {test_name}")
# Define outputs directory and redirect settings to ensure only the latest run is saved
outputs_dir = Path(__file__).resolve().parent / "outputs"
if outputs_dir.exists():
shutil.rmtree(outputs_dir)
outputs_dir.mkdir(parents=True, exist_ok=True)
from artemis.config import settings
settings.TRACES_PATH = outputs_dir
mock_ctx = create_mock_context()
mock_state = create_mock_state()
logger.info(
f"Mock context: device={mock_ctx.device.device_width}x{mock_ctx.device.device_height}"
)
# Load and verify screenshot
input_screenshot = get_or_create_test_screenshot()
logger.info(f"Visual screenshot loaded: {input_screenshot}")
# =========================================================================
# 1. Mock LLM Multi-Turn Responses (Sequentially calling each tool)
# =========================================================================
# Turn 1: call get_ocr_list
mock_fc_ocr = MagicMock()
mock_fc_ocr.name = "get_ocr_list"
mock_fc_ocr.args = {}
mock_resp_ocr = MagicMock()
mock_resp_ocr.function_calls = [mock_fc_ocr]
# Turn 2: call ask_perception_tool
mock_fc_search = MagicMock()
mock_fc_search.name = "ask_perception_tool"
mock_fc_search.args = {
"search_query": "Settings",
"nx": 500,
"ny": 500,
"detect_queries": ["gear icon"],
}
mock_resp_search = MagicMock()
mock_resp_search.function_calls = [mock_fc_search]
# Turn 3: call detect_objects
mock_fc_detect = MagicMock()
mock_fc_detect.name = "detect_objects"
mock_fc_detect.args = {"queries": ["gear icon"], "target_image_id": "img_0"}
mock_resp_detect = MagicMock()
mock_resp_detect.function_calls = [mock_fc_detect]
# Turn 4: call inspect_region
mock_fc_coords = MagicMock()
mock_fc_coords.name = "inspect_region"
mock_fc_coords.args = {
"x_min": 100,
"y_min": 100,
"x_max": 300,
"y_max": 300,
"zoom_factor": 2.0,
}
mock_resp_coords = MagicMock()
mock_resp_coords.function_calls = [mock_fc_coords]
# Turn 5: call submit_answer to complete loop
mock_fc_submit = MagicMock()
mock_fc_submit.name = "submit_answer"
mock_fc_submit.args = {
"candidates": [
{
"label": "T1",
"coords": [462, 208],
"description": "Dashboard Button",
},
{
"label": "S1",
"coords": [462, 208],
"description": "Settings Button",
},
{"label": "D1", "coords": [200, 300], "description": "gear icon"},
],
"fallback_message": ("Successfully verified all visual controls sequentially."),
}
mock_resp_submit = MagicMock()
mock_resp_submit.function_calls = [mock_fc_submit]
mock_client = MagicMock()
mock_client.aio.models.generate_content = AsyncMock(
side_effect=[
mock_resp_ocr,
mock_resp_search,
mock_resp_detect,
mock_resp_coords,
mock_resp_submit,
]
)
mock_ctx._genai_client = mock_client
mock_ctx.llm_config.explorer = None
# =========================================================================
# 2. Mock Downstream Tool Logic Dependencies
# =========================================================================
# Mock SQLite StorageManager for get_ocr_list
mock_storage = MagicMock()
mock_record = MagicMock()
mock_record.ocr_result = [
{"text": "screen description text", "position": []},
{
"text": "Dashboard Button",
"position": [
{"x": 400, "y": 400},
{"x": 600, "y": 400},
{"x": 600, "y": 600},
{"x": 400, "y": 600},
],
},
]
mock_storage.get_image.return_value = mock_record
# Mock low-level layout searching (search_ui_func)
mock_search_ui = MagicMock(
return_value={
"matches": [
{
"matched_text": "Settings Button",
"bounds": [400, 400, 600, 600],
}
]
}
)
# Mock downstream object detection pipeline (_run_object_detection)
mock_detect = AsyncMock(
return_value={"detected": [{"label": "gear icon", "point": [200, 300]}]}
)
# Mock spatial coordinate lookup (search_by_coordinates_func)
mock_lookup = MagicMock(return_value="Matching Elements: Container/DashboardFrame")
logger.info("Starting Explorer.run execution loop with sequential tool mocks...")
with (
patch(
"artemis.agents.explorer.explorer.genai.Client",
return_value=mock_client,
),
patch(
"artemis.agents.explorer.explorer.StorageManager",
return_value=mock_storage,
),
patch("artemis.agents.explorer.explorer.search_ui_func", mock_search_ui),
patch("artemis.agents.explorer.explorer._run_object_detection", mock_detect),
patch(
"artemis.agents.explorer.explorer.search_by_coordinates_func",
mock_lookup,
),
):
explorer = Explorer(mock_ctx)
result = await explorer.run(
query="Test all tools",
context_feedback="",
screenshot_path=str(input_screenshot),
state=mock_state,
version="ultra",
)
# Verify ReAct loop went through all 5 turns
assert mock_client.aio.models.generate_content.call_count == 5
logger.info("Successfully completed all 5 ReAct turns.")
# Retrieve final submission outcome
outcome_data = json.loads(result)
assert "candidates" in outcome_data
assert (
outcome_data["fallback_message"]
== "Successfully verified all visual controls sequentially."
)
logger.info("Submission response verified successfully.")
# Extract all tool outputs from history to assert correctness
call_args_list = mock_client.aio.models.generate_content.call_args_list
# The fifth call receives the accumulated tool history
contents = call_args_list[4].kwargs.get("contents") or call_args_list[4].args[0]
tool_responses = _verify_all_tools_responses(contents, logger)
# ASSERTIONS FOR EACH TOOL INDIVIDUALLY
# Verify each tool execution record in explorer.trace_history
assert len(explorer.trace_history) >= 4
# 1. Verify get_ocr_list
ocr_trace = explorer.trace_history[0]["tool_calls"][0]
assert ocr_trace["name"] == "get_ocr_list"
assert "[T1]" in ocr_trace["response"]["text"]
assert "[462,208]" in ocr_trace["response"]["text"]
assert ocr_trace["response"]["image_path"] is not None
# 2. Verify ask_perception_tool
search_trace = explorer.trace_history[1]["tool_calls"][0]
assert search_trace["name"] == "ask_perception_tool"
assert "XML/OCR Text Search Results" in search_trace["response"]["text"]
assert len(search_trace["response"]["image_paths"]) > 0
# 3. Verify detect_objects
detect_trace = explorer.trace_history[2]["tool_calls"][0]
assert detect_trace["name"] == "detect_objects"
assert "[D" in detect_trace["response"]["text"]
assert detect_trace["response"]["image_path"] is not None
# 4. Verify inspect_region
coords_trace = explorer.trace_history[3]["tool_calls"][0]
assert coords_trace["name"] == "inspect_region"
assert coords_trace["response"]["image_path"] is not None
# Compute expected screenshot hash
import hashlib
sha256_hash = hashlib.sha256()
with open(input_screenshot, "rb") as f:
for byte_block in iter(lambda: f.read(4096), b""):
sha256_hash.update(byte_block)
expected_hash = sha256_hash.hexdigest()
# Should map normalized [500, 500] to pixel [540, 1200]
mock_lookup.assert_called_once_with(expected_hash, 540, 1200)
logger.info(
"All assertions passed! The entire suite of explorer tools is fully functional."
)
# Clean up logging handlers
agent_logger.removeHandler(handler)
logger.removeHandler(handler)
handler.close()
if __name__ == "__main__":
asyncio.run(test_explorer_all_tools_sequential_mocked())
@@ -0,0 +1,372 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import json
import logging
from pathlib import Path
import sys
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.agents.explorer.explorer import Explorer
from tests.integration.agents.explorer.test_explorer_all_tools.helpers import (
create_mock_context,
create_mock_state,
get_or_create_test_screenshot,
)
from PIL import Image
import pytest
def _verify_tool_responses(contents: list, logger: logging.Logger):
"""Parses the multi-turn conversational history payload and maps tool outputs."""
logger.info("=== VERIFYING TOOL RESPONSES IN CONVERSATION HISTORY ===")
tool_responses = {}
for content in contents:
for part in content.parts:
if part.function_response:
name = part.function_response.name
response_data = part.function_response.response
# Extract result text
tool_text = None
if hasattr(response_data, "get"):
tool_text = response_data.get("result")
elif hasattr(response_data, "fields"):
tool_text = response_data.fields.get("result")
else:
tool_text = getattr(response_data, "result", None)
if name not in tool_responses:
tool_responses[name] = []
tool_responses[name].append({"text": tool_text, "has_image": False})
logger.info(f"Found tool response for '{name}':\n{tool_text}")
if part.inline_data and part.inline_data.mime_type == "image/jpeg":
if tool_responses:
last_tool = list(tool_responses.keys())[-1]
tool_responses[last_tool][-1]["has_image"] = True
tool_responses[last_tool][-1]["img_bytes"] = part.inline_data.data
logger.info(
f" -> Attached {len(part.inline_data.data)} bytes of"
f" JPEG inline visual content to '{last_tool}'"
)
return tool_responses
@pytest.mark.asyncio
async def test_explorer_ask_image_processor_tool():
"""Verify that the Explorer ReAct loop correctly dispatches, runs, processes, and
handles the output image/transform details for the 'ask_image_processor'
tool.
The test ends immediately after the Vision Coder responds (in the second
turn).
"""
test_name = sys._getframe().f_code.co_name
# Set up standard log file saved on the same level as test file
log_file = Path(__file__).parent / f"log_{test_name}.log"
logger = logging.getLogger(test_name)
logger.setLevel(logging.INFO)
handler = logging.FileHandler(log_file, mode="w", encoding="utf-8")
logger.addHandler(handler)
# Attach the main agent logger to capture internal step logs
agent_logger = logging.getLogger("artemis.agents.explorer.explorer")
agent_logger.setLevel(logging.INFO)
agent_logger.addHandler(handler)
logger.info(f"Starting test: {test_name}")
# Define outputs directory and redirect settings to ensure only the latest run is saved
outputs_dir = Path(__file__).resolve().parent / "outputs"
if not outputs_dir.exists():
outputs_dir.mkdir(parents=True, exist_ok=True)
from artemis.config import settings
settings.TRACES_PATH = outputs_dir
mock_ctx = create_mock_context()
mock_state = create_mock_state()
logger.info(
f"Mock context: device={mock_ctx.device.device_width}x{mock_ctx.device.device_height}"
)
# Load and verify screenshot
input_screenshot = get_or_create_test_screenshot()
logger.info(f"Visual screenshot loaded: {input_screenshot}")
# =========================================================================
# 1. Mock LLM Multi-Turn Responses (ask_image_processor -> submit_answer)
# =========================================================================
# Turn 1: call ask_image_processor
mock_fc_coder = MagicMock()
mock_fc_coder.name = "ask_image_processor"
mock_fc_coder.args = {
"instruction": (
"Crop the top-right area, convert to HSV, resize, then convert to grayscale."
),
"target_image_id": "img_0",
}
mock_resp_coder = MagicMock()
mock_resp_coder.function_calls = [mock_fc_coder]
# Turn 2: call submit_answer to complete loop immediately after vision coder responds
mock_fc_submit = MagicMock()
mock_fc_submit.name = "submit_answer"
mock_fc_submit.args = {
"candidates": [],
"fallback_message": ("Successfully processed the image via Vision Coder."),
}
mock_resp_submit = MagicMock()
mock_resp_submit.function_calls = [mock_fc_submit]
mock_client = MagicMock()
mock_client.aio.models.generate_content = AsyncMock(
side_effect=[mock_resp_coder, mock_resp_submit]
)
mock_ctx._genai_client = mock_client
mock_ctx.llm_config.explorer = None
# =========================================================================
# 2. Mock ImageProcessor agent
# =========================================================================
# Create the mock output image path
mock_new_image_path = outputs_dir / f"mock_image_processor_output_{test_name}.jpg"
# Create a small dummy image of size 100x100 using PIL so that cv2.imread inside explorer reads these dimensions
img = Image.new("RGB", (100, 100), color="red")
img.save(mock_new_image_path)
# Also write a transform JSON file for the new image
transform_path = Path(str(mock_new_image_path) + ".transform.json")
mock_transform_data = {
"offset_x": 108.0,
"offset_y": 240.0,
"scale_x": 0.5,
"scale_y": 0.5,
}
with open(transform_path, "w") as f:
json.dump(mock_transform_data, f)
# We mock ImageProcessor class itself
mock_image_processor_instance = MagicMock()
mock_image_processor_instance.run = AsyncMock(
return_value={
"outputs": [
{
"image_id": "img_1",
"path": str(mock_new_image_path),
"transform": {
"offset_x": 108.0,
"offset_y": 240.0,
"scale_x": 0.5,
"scale_y": 0.5,
},
"annotations": {"V1": [50, 100], "V2": [180, 240]},
}
],
"summary": "Cropped and highlighted the red button",
}
)
logger.info("Starting Explorer.run execution loop with ask_image_processor...")
with (
patch(
"artemis.agents.explorer.explorer.genai.Client",
return_value=mock_client,
),
patch(
"artemis.agents.image_processor.image_processor.ImageProcessor",
return_value=mock_image_processor_instance,
),
):
explorer = Explorer(mock_ctx)
result = await explorer.run(
query=("Click on the red button in the top-right corner of the screen."),
context_feedback="",
screenshot_path=str(input_screenshot),
state=mock_state,
version="ultra",
)
# Verify initial image pool has img_0
assert "img_0" in explorer.image_pool
assert explorer.image_pool["img_0"]["path"] == str(input_screenshot)
# Verify ReAct loop went through exactly 2 turns
assert mock_client.aio.models.generate_content.call_count == 2
logger.info("Successfully completed both ReAct turns.")
# Verify ImageProcessor run was called with correct parameters
mock_image_processor_instance.run.assert_called_once_with(
"Crop the top-right area, convert to HSV, resize, then convert to grayscale.",
str(input_screenshot),
)
logger.info("Verified ImageProcessor was invoked with correct parameters.")
# Verify new image is registered in explorer's image_pool
assert "img_1" in explorer.image_pool
img_1_info = explorer.image_pool["img_1"]
assert img_1_info["path"] == str(mock_new_image_path)
# Verify transform calculations:
# Since parent (img_0) transform is {"offset_x": 0.0, "offset_y": 0.0, "scale_x": 1.0, "scale_y": 1.0}
# and child transform is {"offset_x": 108.0, "offset_y": 240.0, "scale_x": 0.5, "scale_y": 0.5}
assert img_1_info["transform"]["scale_x"] == 0.5
assert img_1_info["transform"]["scale_y"] == 0.5
assert img_1_info["transform"]["offset_x"] == 108.0
assert img_1_info["transform"]["offset_y"] == 240.0
assert "Cropped and highlighted the red button" in img_1_info["description"]
logger.info("Verified image_pool registrations and coordinate transforms.")
# Retrieve final submission outcome
outcome_data = json.loads(result)
assert (
outcome_data["fallback_message"] == "Successfully processed the image via Vision Coder."
)
logger.info("Submission response verified successfully.")
# Extract all tool outputs from history to assert correctness
call_args_list = mock_client.aio.models.generate_content.call_args_list
# The second call receives the accumulated tool history
contents = call_args_list[1].kwargs.get("contents") or call_args_list[1].args[0]
tool_responses = _verify_tool_responses(contents, logger)
# Assertions for ask_image_processor tool response (Turn 1 output)
assert "ask_image_processor" in tool_responses
coder_resp = tool_responses["ask_image_processor"][0]
assert "Image Processor completed successfully" in coder_resp["text"]
assert "img_1" in coder_resp["text"]
assert "Annotations (wrt original screenshot):" in coder_resp["text"]
assert "img_1:" in coder_resp["text"]
assert "- [V1]: [208, 440]" in coder_resp["text"]
assert "- [V2]: [468, 720]" in coder_resp["text"]
assert coder_resp["has_image"]
assert len(coder_resp["img_bytes"]) > 0
logger.info(
"Verified tool response contains the response text, annotations"
" list, and the attached image bytes."
)
# Clean up dummy files (Commented out temporarily so you can see the images!)
# if mock_new_image_path.exists():
# mock_new_image_path.unlink()
# if transform_path.exists():
# transform_path.unlink()
# if input_screenshot.exists():
# input_screenshot.unlink()
# Clean up logging handlers
agent_logger.removeHandler(handler)
logger.removeHandler(handler)
handler.close()
if __name__ == "__main__":
asyncio.run(test_explorer_ask_image_processor_tool())
def test_cv_canvas_annotations(tmp_path):
import numpy as np
import cv2
from artemis.utils.cv_canvas import ImageCanvas
# Create a dummy image
img_dir = tmp_path / "images"
img_dir.mkdir()
img_path = img_dir / "img_0.jpg"
img_data = np.zeros((100, 100, 3), dtype=np.uint8)
cv2.imwrite(str(img_path), img_data)
# Initialize transforms registry
registry_path = img_dir / "intermediate_transforms.json"
registry_data = {
"img_0": {
"image_id": "img_0",
"path": str(img_path.resolve()),
"transform": {
"offset_x": 0.0,
"offset_y": 0.0,
"scale_x": 1.0,
"scale_y": 1.0,
},
"annotations": {},
}
}
with open(registry_path, "w") as f:
json.dump(registry_data, f)
canvas = ImageCanvas("img_0", str(img_dir))
assert canvas.annotations == {}
# Test _get_resolution and initial width/height
assert canvas._get_resolution() == (100, 100)
assert canvas.width == 100
assert canvas.height == 100
# Test normalized_to_pixel_coords
assert canvas.normalized_to_pixel_coords(500, 500) == (50, 50)
assert canvas.normalized_to_pixel_coords(0, 0) == (0, 0)
assert canvas.normalized_to_pixel_coords(1000, 1000) == (99, 99)
# Draw dot
canvas.draw_dot(10, 20, 1)
assert canvas.annotations == {"V1": [10, 20]}
# Crop
canvas.crop(5, 5, 20, 20)
assert canvas.width == 20
assert canvas.height == 20
assert canvas._get_resolution() == (20, 20)
assert canvas.normalized_to_pixel_coords(500, 500) == (10, 10)
assert canvas.annotations == {"V1": [5, 15]}
# Out of bounds test
canvas.draw_dot(2, 2, 2)
assert canvas.annotations == {"V1": [5, 15], "V2": [2, 2]}
# Crop again to discard V1
canvas.crop(1, 1, 5, 5)
assert canvas.width == 5
assert canvas.height == 5
assert canvas.normalized_to_pixel_coords(500, 500) == (2, 2)
# V1 was at [5, 15], after crop(1, 1), it becomes [4, 14], which is out of bounds of 5x5.
# V2 was at [2, 2], after crop(1, 1), it becomes [1, 1], which is inside 5x5.
assert canvas.annotations == {"V2": [1, 1]}
# Resize by factor
canvas.resize_by_factor(2.0)
assert canvas.width == 10
assert canvas.height == 10
assert canvas._get_resolution() == (10, 10)
assert canvas.normalized_to_pixel_coords(500, 500) == (5, 5)
assert canvas.annotations == {"V2": [2, 2]}
# Save and verify registry
canvas.save(final=True)
with open(registry_path) as f:
saved_registry = json.load(f)
assert "img_1" in saved_registry
assert saved_registry["img_1"]["annotations"] == {"V2": [2, 2]}
assert saved_registry["img_1"]["is_output"] is True
@@ -0,0 +1,232 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import json
import logging
from pathlib import Path
import shutil
from unittest.mock import patch
from artemis.config import settings
from artemis.agents.image_processor.image_processor import ImageProcessor
from tests.integration.agents.explorer.test_explorer_all_tools.helpers import (
create_mock_context,
get_or_create_test_screenshot,
)
from artemis.utils.python_executor import PythonExecutor
import pytest
# Setup logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("test_explorer_ask_image_processor_jupyter")
@pytest.mark.asyncio
async def test_explorer_ask_image_processor_jupyter():
test_name = "test_explorer_ask_image_processor_jupyter"
# 1. Setup outputs directory
outputs_dir = Path(__file__).resolve().parent / "outputs" / test_name
outputs_dir.mkdir(parents=True, exist_ok=True)
# Redirect traces path to the new outputs directory
settings.TRACES_PATH = outputs_dir
# 2. Prepare the input image
src_image_path = get_or_create_test_screenshot()
input_screenshot = outputs_dir / "input_screenshot.jpg"
shutil.copy2(src_image_path, input_screenshot)
logger.info(f"Copied input screenshot to: {input_screenshot}")
# 3. Formulate the instruction with the injected G-Logo coordinate prior
instruction = (
'The user wants to find "the red patch above the green patch". Let\'s'
" analyze the screen layout. \nIn the dock / app icons, we have:\n-"
" Play Store (9): colorful triangle (has green, blue, yellow, red)\n-"
" Gmail (10): M icon (red, yellow, green, blue)\n- Photos (11):"
" pinwheel (red, yellow, green, blue)\n- Chrome (16): circular logo"
" with green on the bottom, red on top, yellow on the side, blue"
" center.\nWait, let's look at the Photos icon (11) or Chrome icon (16)"
" or Gmail (10) or Google search bar logo (21) or Play Store"
" (9).\nLet's zoom into Chrome (16), Gmail (10), Photos (11), Play"
" Store (9), Google logo (21) to analyze the relative positions of red"
" patches and green patches.\nThe G-Logo is exactly at pixel"
" coordinates [2173, 107, 2241, 174].\nLet's crop these icons to find"
' which icon has a "red patch above the green patch" or is it a'
" specific patch?\nWait!\nLet's look at Chrome (16):\nRed is at the"
" top, green is at the bottom/left. Red is directly above green? Yes,"
" the top slice of Chrome is red, and the bottom slice is green.\nLet's"
" look at Photos (11):\nIt has four petals. The top petal is red? No,"
" top petal is red/pink, bottom is green? Let's check.\nLet's look at"
" Gmail (10):\nThe red is the M shape, green is at the bottom"
" right?\nLet's write Python code to crop these areas and output the"
" crop to see clearly.\nSpecifically, let's crop:\n- Play Store (bounds"
" around 9)\n- Gmail (bounds around 10)\n- Photos (bounds around 11)\n-"
" Chrome (bounds around 16)\n- Google widget logo (bounds around"
" 21)\nWe will save a visualization of these cropped regions side by"
" side or separately so we can examine the colors."
)
# 4. Intercept Python executions to record them for the Jupyter Notebook
executed_turns = []
original_execute = PythonExecutor.execute
def wrapped_execute(self, code: str) -> str:
logger.info(f"Intercepting Python code execution. Code:\n{code}\n")
result = original_execute(self, code)
logger.info(f"Execution result:\n{result}\n")
executed_turns.append({"code": code, "output": result})
return result
from langchain_core.messages import AIMessage
from unittest.mock import AsyncMock, MagicMock
sample_code = """
from artemis.utils.cv_canvas import ImageCanvas
import glob
from pathlib import Path
from artemis.config import settings
dirs = sorted(glob.glob(str(Path(settings.TRACES_PATH) / "images" / "image_processor" / "intermediates_*")))
intermediate_dir = dirs[-1] if dirs else "."
canvas = ImageCanvas("img_0", intermediate_dir)
canvas.crop(10, 10, 50, 50)
canvas.save(final=True)
"""
mock_llm = MagicMock()
mock_bound = MagicMock()
mock_bound.ainvoke = AsyncMock(
side_effect=[
AIMessage(
content="",
tool_calls=[
{
"name": "execute_python",
"args": {"code": sample_code},
"id": "call_1",
}
],
),
AIMessage(
content="",
tool_calls=[
{
"name": "submit_result",
"args": {"summary": "Cropped target G-Logo region."},
"id": "call_2",
}
],
),
]
)
mock_llm.bind_tools.return_value = mock_bound
mock_ctx = create_mock_context()
image_processor = ImageProcessor(mock_ctx)
# Run with patched executor and mock LLM
logger.info("Running Image Processor with G-Logo prior...")
with (
patch.object(PythonExecutor, "execute", wrapped_execute),
patch(
"artemis.agents.image_processor.image_processor.get_llm",
return_value=mock_llm,
),
):
result = await image_processor.run(instruction, str(input_screenshot))
logger.info(f"Image Processor finished. Result: {result}")
# 5. Compile and save the Jupyter Notebook (.ipynb)
notebook_cells = []
# Add a markdown introduction cell
notebook_cells.append(
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Image Processor Execution Trace\n",
f"**Task**: {instruction[:150]}...\n",
"**Input Image**: `input_screenshot.jpg`",
],
}
)
for turn_idx, turn in enumerate(executed_turns):
# Add code cell
notebook_cells.append(
{
"cell_type": "code",
"execution_count": turn_idx + 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [line + "\n" for line in turn["output"].split("\n")],
}
],
"source": [line + "\n" for line in turn["code"].split("\n")],
}
)
# If a result was submitted, add a final markdown cell
outputs = result.get("outputs", [])
if outputs:
final_image_path = outputs[-1]["path"]
notebook_cells.append(
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 🎉 Execution Completed Successfully\n",
f"**Summary**: {result.get('summary')}\n",
f"**Output Image**: `{Path(final_image_path).name}`",
],
}
)
# Copy final image to output directory root for easy access
shutil.copy2(final_image_path, outputs_dir / "final_output.jpg")
notebook_json = {
"cells": notebook_cells,
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3",
}
},
"nbformat": 4,
"nbformat_minor": 2,
}
notebook_path = outputs_dir / "vision_coder_execution.ipynb"
with open(notebook_path, "w", encoding="utf-8") as f:
json.dump(notebook_json, f, indent=2)
logger.info(f"Saved compiled Jupyter notebook to: {notebook_path}")
# 6. Assertions
assert "error" not in result, f"Image Processor failed: {result.get('error')}"
assert "outputs" in result, "Image Processor did not return outputs"
assert len(result["outputs"]) > 0, "Image Processor did not return any output images"
logger.info("Test completed successfully! All assertions passed.")
if __name__ == "__main__":
asyncio.run(test_explorer_ask_image_processor_jupyter())
@@ -0,0 +1,130 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import time
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.agents.object_detector.object_detector import _run_object_detection
from tests.integration.agents.explorer.test_explorer_all_tools.helpers import create_mock_context
import pytest
@pytest.mark.asyncio
async def test_object_detector_concurrency_and_throttling(tmp_path):
"""Verify that object detector requests run in parallel and are correctly throttled by the semaphore."""
# Create a dummy image file for the test
dummy_image_path = tmp_path / "dummy.jpg"
dummy_image_path.write_bytes(b"dummy image data")
# Setup mock context
mock_ctx = create_mock_context()
# Configure the object detector LLM config explicitly to avoid MagicMocks for timeout and model
od_cfg = MagicMock()
od_cfg.model = "gemini-robotics-er-early-access"
od_cfg.fix_model = "gemini-2.5-flash"
od_cfg.timeout = 10.0
mock_ctx.llm_config.utils.object_detector = od_cfg
# Track active requests
active_requests = 0
max_concurrent_requests = 0
request_lock = asyncio.Lock()
from langchain_core.messages import AIMessage
mock_llm = MagicMock()
mock_llm.ainvoke = AsyncMock(side_effect=lambda *args, **kwargs: mock_generate_content())
async def mock_generate_content(*args, **kwargs):
nonlocal active_requests, max_concurrent_requests
async with request_lock:
active_requests += 1
if active_requests > max_concurrent_requests:
max_concurrent_requests = active_requests
try:
await asyncio.sleep(0.1)
finally:
async with request_lock:
active_requests -= 1
return AIMessage(content='[{"point": [500, 500]}]')
mock_llm.ainvoke = AsyncMock(side_effect=mock_generate_content)
with patch(
"artemis.agents.object_detector.object_detector.get_llm",
return_value=mock_llm,
):
# --- TEST 1: Concurrency Assertion (5 labels) ---
# 5 labels should run in parallel, completing in ~0.1s (much faster than a sequential 0.5s)
# Reset trackers
active_requests = 0
max_concurrent_requests = 0
start_time = time.time()
labels_5 = [f"label_{i}" for i in range(5)]
templates = ["Find {labels_str}"]
result_5 = await _run_object_detection(
ctx=mock_ctx,
image_path=str(dummy_image_path),
queries=labels_5,
templates=templates,
)
elapsed_5 = time.time() - start_time
# Concurrency assertions
assert max_concurrent_requests == 5, (
f"Expected 5 concurrent requests, got {max_concurrent_requests}"
)
assert elapsed_5 < 0.3, f"Expected execution to take < 0.3s, but took {elapsed_5:.3f}s"
# Verify result structure and swapped coordinates [y, x] -> [x, y]
parsed_result_5 = result_5
assert len(parsed_result_5["detected"]) == 5
for item in parsed_result_5["detected"]:
assert item["point"] == [500, 500]
# --- TEST 2: Throttling Assertion (10 labels) ---
# 10 labels should be throttled by the semaphore limit of 6.
# It should run in two batches, completing in ~0.2s (much faster than a sequential 1.0s).
# Reset trackers
active_requests = 0
max_concurrent_requests = 0
start_time = time.time()
labels_10 = [f"label_{i}" for i in range(10)]
result_10 = await _run_object_detection(
ctx=mock_ctx,
image_path=str(dummy_image_path),
queries=labels_10,
templates=templates,
)
elapsed_10 = time.time() - start_time
# Throttling assertions
assert max_concurrent_requests == 6, (
f"Expected max concurrent requests to be capped at 6, got {max_concurrent_requests}"
)
assert elapsed_10 < 0.5, (
f"Expected execution to take < 0.5s (two batches of 0.1s), but took {elapsed_10:.3f}s"
)
parsed_result_10 = result_10
assert len(parsed_result_10["detected"]) == 10
for item in parsed_result_10["detected"]:
assert item["point"] == [500, 500]
@@ -0,0 +1,126 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import base64
import logging
import os
import sys
from dotenv import load_dotenv
# Ensure we can import from project
sys.path.append(os.path.dirname(__file__))
from artemis.sdk import Agent
from artemis.sdk.builders import Builders
from artemis.sdk.types import AgentProfile
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.agents.validator.failure_analyzer import FailureAnalyzer, ValidationErrorCategory
from artemis.controllers.unified_controller import UnifiedMobileController
from artemis.utils.notes import get_note_file_path
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("test_fa_standalone")
async def main():
load_dotenv()
# 1. Initialize Agent to set up device and connection pools
logger.info("Initializing Agent and connecting to device...")
profile = AgentProfile(name="default", from_file="llm-config.override.jsonc")
config = Builders.AgentConfig.with_default_profile(profile).build()
agent = Agent(config=config)
await agent.init()
# Create the ArtemisContext manually using agent's initialized clients
context = ArtemisContext(
device=agent._device_context,
adb_client=agent._adb_client,
ui_adb_client=agent._ui_adb_client,
llm_config=profile.llm_config,
agent_config=config,
)
# Set up trace session and data engine
from artemis.data_engine.engine import DataEngine
data_engine = DataEngine(ctx=context)
context.data_engine = data_engine
data_engine.start_session(goal="Play a video in full screen on YouTube in Chrome")
# Write the task plan note
task_plan_content = (
"- [ ] Open Chrome and navigate to the YouTube website (youtube.com)\n"
"- [ ] Select and play any video\n"
"- [ ] Make the playing video full screen"
)
task_plan_path = get_note_file_path(data_engine.base_dir, "task_plan")
task_plan_path.parent.mkdir(parents=True, exist_ok=True)
task_plan_path.write_text(task_plan_content, encoding="utf-8")
# 2. Get initial screenshot
controller = UnifiedMobileController(context)
screen_data = await controller.controller.get_screen_data()
screenshot_b64 = screen_data.base64
# Set up State
state = State(
messages=[],
initial_goal="Play a video in full screen on YouTube in Chrome",
latest_ui_hierarchy=screen_data.elements,
latest_screenshot=str(
data_engine.get_image_path(
data_engine.get_or_create_image(base64.b64decode(screenshot_b64))
)
),
focused_app_info="",
device_date="",
structured_decisions="",
complete_subgoals_by_ids=[],
validator_messages=[],
)
# 3. Instantiate and run Failure Analyzer directly
logger.info("Starting Failure Analyzer directly...")
analyzer = FailureAnalyzer(context)
analyzer.max_iterations = 45
# We pass a dummy failed action representing the fullscreen button tap
dummy_failed_action = {
"action": "tap",
"coordinates": [968, 883],
"target_text": "fullscreen button",
}
try:
result = await analyzer.analyze(
state=state,
failed_action=dummy_failed_action,
error_msg="Simulated launch failure to trigger standalone FA run",
pre_screenshot=screenshot_b64,
post_screenshot=screenshot_b64,
error_category=ValidationErrorCategory.TARGET_DISAPPEARED,
)
logger.info(f"Failure Analyzer direct execution finished! Result: {result}")
except Exception:
logger.exception("Error during Failure Analyzer execution:")
finally:
await agent.clean()
if __name__ == "__main__":
asyncio.run(main())
@@ -0,0 +1,58 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import logging
import os
import sys
from dotenv import load_dotenv
# Ensure we can import from project
sys.path.append(os.path.dirname(__file__))
from artemis.sdk import Agent
from artemis.sdk.builders import Builders
from artemis.sdk.types import AgentProfile
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("test_flash_standalone")
async def main():
load_dotenv()
# 1. Initialize Agent to set up device and connection pools
logger.info("Initializing Agent and connecting to device...")
profile = AgentProfile(name="default", from_file="llm-config.override.jsonc")
config = Builders.AgentConfig.with_default_profile(profile).build()
agent = Agent(config=config)
await agent.init()
# 2. Run task using 'flash' profile
logger.info("Starting Flash runner task execution...")
try:
result = await agent.run_task(
goal="Open USA Today in Chrome browser and view the USA news",
profile="flash",
)
logger.info(f"Flash runner task execution finished! Result: {result}")
except Exception:
logger.exception("Error during Flash task execution:")
finally:
await agent.clean()
if __name__ == "__main__":
asyncio.run(main())
@@ -0,0 +1,57 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import logging
import os
import sys
from dotenv import load_dotenv
# Ensure we can import from project
sys.path.append(os.path.dirname(__file__))
from artemis.sdk import Agent
from artemis.sdk.builders import Builders
from artemis.sdk.types import AgentProfile
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("test_flash_standalone")
async def main():
load_dotenv()
# 1. Initialize Agent to set up device and connection pools
logger.info("Initializing Agent and connecting to device...")
profile = AgentProfile(name="default", from_file="llm-config.override.jsonc")
config = Builders.AgentConfig.with_default_profile(profile).build()
agent = Agent(config=config)
await agent.init()
# 2. Run task using 'flash' profile
logger.info("Starting Flash runner task execution...")
try:
result = await agent.run_task(
goal="Play a video in full screen on YouTube in Chrome", profile="flash"
)
logger.info(f"Flash runner task execution finished! Result: {result}")
except Exception:
logger.exception("Error during Flash task execution:")
finally:
await agent.clean()
if __name__ == "__main__":
asyncio.run(main())
+21
View File
@@ -0,0 +1,21 @@
import time
import uiautomator2 as u2
def test():
device = u2.connect("emulator-5554")
print("Testing send_keys with immediate IME switch off")
device.set_fastinput_ime(True)
device.send_keys("Test 1")
device.set_fastinput_ime(False)
time.sleep(2)
print("Testing send_keys with delay before IME switch off")
device.set_fastinput_ime(True)
device.send_keys("Test 2")
time.sleep(1.0)
device.set_fastinput_ime(False)
if __name__ == "__main__":
test()
+102
View File
@@ -0,0 +1,102 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
from pathlib import Path
import sys
# Add root to sys.path
root_dir = Path(__file__).parent.parent
if str(root_dir) not in sys.path:
sys.path.insert(0, str(root_dir))
from adbutils import AdbClient
from artemis.clients.ui_automator_client import UIAutomatorClient
from artemis.context import DeviceContext, DevicePlatform, ArtemisContext
from artemis.controllers.unified_controller import UnifiedMobileController
import pytest
@pytest.mark.asyncio
async def test_actions():
print("=========================================")
print(" Physical Action Direct Validation Testbench")
print("=========================================")
adb = AdbClient(host="localhost", port=5037)
devices = adb.device_list()
if not devices:
print("❌ Error: No active Android emulator or physical device detected.")
return
device_id = devices[0].serial
print(f"✅ Target device locked: {device_id}")
ui_client = UIAutomatorClient(device_id=device_id)
print("🔄 Connecting to UIAutomator2 and syncing screen properties...")
try:
ui_data = ui_client.get_screen_data()
w, h = ui_data.width, ui_data.height
print(f"✅ Screen sync ready, physical resolution: {w}x{h}")
except Exception as e:
print(f"❌ Screen properties sync failed: {e}")
return
ctx = ArtemisContext(
trace_id="physical-test-bench",
device=DeviceContext(
host_platform="LINUX",
mobile_platform=DevicePlatform.ANDROID,
device_id=device_id,
device_width=w,
device_height=h,
),
adb_client=adb,
ui_adb_client=ui_client,
)
ctrl = UnifiedMobileController(ctx)
print("\n[Task 1] Extract Live UI Hierarchy...")
try:
elems = await ctrl.get_ui_elements()
print(f"✅ Parsed UI element count: {len(elems)}")
except Exception as e:
print(f"❌ Failed to extract UI hierarchy: {e}")
print("\n[Task 2] Capture Live Screenshot...")
try:
shot = await ctrl.take_screenshot()
print(f"✅ Screenshot captured, Base64 payload length: {len(shot)}")
except Exception as e:
print(f"❌ Failed to capture screenshot: {e}")
print("\n[Task 3] Simulate Drag/Swipe Down...")
try:
# Swipe up from bottom to top
err = await ctrl.swipe_coords(w // 2, int(h * 0.8), w // 2, int(h * 0.2), duration=400)
if err:
print(f"❌ Swipe rejected by controller: {err}")
else:
print("✅ Swipe command injected successfully!")
except Exception as e:
print(f"❌ Swipe execution exception: {e}")
print("\n🧹 Safely releasing connection and driver handles...")
await ctrl.cleanup()
print("🎉 Direct physical validation tests completed successfully!")
if __name__ == "__main__":
asyncio.run(test_actions())
@@ -0,0 +1,45 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Integration tests for ReactiveRunner and Pipeline."""
import pytest
from artemis.core.context import ExecutionContext
from artemis.core.state import ExecutionStatus
from artemis.drivers.mock.mock_driver import MockDeviceDriver
from artemis.engine.pipeline import Pipeline
from artemis.engine.reactive_runner import ReactiveRunner
@pytest.mark.asyncio
async def test_reactive_runner_execution():
"""Verify ReactiveRunner executes workflow cycles on mock driver."""
ctx = ExecutionContext(task_goal="Turn on Bluetooth in Settings", device_id="test-emulator")
driver = MockDeviceDriver(device_id="test-emulator")
runner = ReactiveRunner(ctx=ctx, driver=driver)
state = await runner.run(max_turns=5)
assert state.status == ExecutionStatus.SUCCESS
assert state.current_turn >= 1
assert len(state.steps) >= 1
assert state.steps[0].action_name == "wait_for_delay"
@pytest.mark.asyncio
async def test_pipeline_dispatch():
"""Verify Pipeline executes with default mock driver."""
state = await Pipeline.execute(goal="Open YouTube and play video", profile="flash")
assert state.status == ExecutionStatus.SUCCESS
assert state.task_goal == "Open YouTube and play video"