test: add comprehensive unit, integration, and e2e test suites
Introduce complete test coverage for the ARTEMIS platform across multiple layers: Unit tests: Add tests for agent nodes (Planner, Operator, Validator, Diagnoser, FlashRunner, Explorer), tools, utils (OCR/XML fusion, UI hierarchy), and data engine. Integration tests: Add live tool verification, reactive runner, IME, and standalone workflow tests. E2E tests: Add end-to-end scenario tests for media playback and failure recovery flows. Fixtures & mocks: Add mock inputs (audio, video, screenshot) and test utilities.
This commit is contained in:
1 parent
c1803fcb86
commit
5d2283deaa
115 files changed
+22813
No files matched your search
@@ -0,0 +1,68 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Helper utilities for Explorer integration tests."""
|
||||
|
||||
from pathlib import Path
|
||||
from PIL import Image
|
||||
from unittest.mock import MagicMock
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
|
||||
|
||||
def create_mock_context() -> MagicMock:
|
||||
"""Creates a mock ArtemisContext suitable for Explorer tests."""
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.device = MagicMock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
ctx.llm_config = MagicMock()
|
||||
ctx.agent_config = MagicMock()
|
||||
ctx.agent_config.blacklisted_tools = {"explorer": []}
|
||||
ctx.data_engine = None
|
||||
ctx.adb_client = None
|
||||
return ctx
|
||||
|
||||
|
||||
def create_mock_state() -> MagicMock:
|
||||
"""Creates a mock State suitable for Explorer tests."""
|
||||
state = MagicMock(spec=State)
|
||||
state.operator_raw_data = {"width": 1080, "height": 2400}
|
||||
state.latest_ui_hierarchy = []
|
||||
state.latest_screenshot = None
|
||||
return state
|
||||
|
||||
|
||||
def get_or_create_test_screenshot(preferred_path: Path | None = None) -> Path:
|
||||
"""Finds an existing test screenshot or generates a valid 1080x2400 mock JPEG."""
|
||||
if preferred_path and preferred_path.exists() and preferred_path.stat().st_size > 0:
|
||||
return preferred_path
|
||||
|
||||
# Search known repo screenshot paths
|
||||
candidate_paths = [
|
||||
Path(__file__).parent / "input_screenshot_test_explorer_all_tools_sequential_mocked.jpg",
|
||||
Path(__file__).parent / "input_screenshot_test_explorer_ask_vision_coder_tool.jpg",
|
||||
Path(__file__).parents[3] / "tools" / "inputs" / "screenshot.jpg",
|
||||
]
|
||||
for p in candidate_paths:
|
||||
if p.exists() and p.stat().st_size > 0:
|
||||
return p
|
||||
|
||||
# If none found, generate a mock RGB image
|
||||
out_path = (
|
||||
Path(__file__).parent / "input_screenshot_test_explorer_all_tools_sequential_mocked.jpg"
|
||||
)
|
||||
img = Image.new("RGB", (1080, 2400), color=(73, 109, 137))
|
||||
img.save(out_path, format="JPEG")
|
||||
return out_path
|
||||
BIN
Binary file not shown.
|
After Width: | Height: | Size: 848 KiB |
BIN
Binary file not shown.
|
After Width: | Height: | Size: 848 KiB |
@@ -0,0 +1,280 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.agents.explorer.explorer import Explorer
|
||||
from tests.integration.agents.explorer.test_explorer_all_tools.helpers import (
|
||||
create_mock_context,
|
||||
create_mock_state,
|
||||
)
|
||||
from tests.integration.agents.explorer.test_explorer_all_tools.seed_mock_image import (
|
||||
seed_database,
|
||||
)
|
||||
|
||||
# Set up logging to stdout
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
logger = logging.getLogger("live_tools_test")
|
||||
|
||||
|
||||
def extract_and_print_tool_results(contents, set_idx, logger):
|
||||
logger.info("\n======================================================================")
|
||||
logger.info(f" RESULTS FOR INPUT TEST SET {set_idx}")
|
||||
logger.info("======================================================================")
|
||||
|
||||
for content in contents:
|
||||
if content.role != "tool":
|
||||
continue
|
||||
for part in content.parts:
|
||||
if part.function_response:
|
||||
name = part.function_response.name
|
||||
response_data = part.function_response.response
|
||||
|
||||
# Extract result text
|
||||
tool_text = None
|
||||
if hasattr(response_data, "get"):
|
||||
tool_text = response_data.get("result")
|
||||
elif hasattr(response_data, "fields"):
|
||||
tool_text = response_data.fields.get("result")
|
||||
else:
|
||||
tool_text = getattr(response_data, "result", None)
|
||||
|
||||
logger.info(f"\n🛠️ Tool: '{name}'")
|
||||
logger.info("--------------------------------------------------")
|
||||
logger.info(f"Output Result:\n{tool_text}")
|
||||
|
||||
if part.inline_data and part.inline_data.mime_type == "image/jpeg":
|
||||
logger.info(
|
||||
f"📷 Visual Data: Attached {len(part.inline_data.data)}"
|
||||
" bytes of annotated JPEG screenshot"
|
||||
)
|
||||
|
||||
|
||||
async def run_single_set(set_idx, query, search_query, labels, coords_nx, coords_ny, candidates):
|
||||
import os
|
||||
|
||||
os.environ["GOOGLE_API_KEY"] = "dummy_api_key"
|
||||
|
||||
mock_ctx = create_mock_context()
|
||||
mock_ctx.llm_config.utils = MagicMock()
|
||||
mock_ctx.llm_config.utils.object_detector = MagicMock()
|
||||
mock_ctx.llm_config.utils.object_detector.model = "gemini-2.5-flash"
|
||||
mock_ctx.llm_config.utils.object_detector.timeout = 10.0
|
||||
|
||||
mock_state = create_mock_state()
|
||||
|
||||
screenshot_path = (
|
||||
Path(__file__).resolve().parent
|
||||
/ "input_screenshot_test_explorer_all_tools_sequential_mocked.jpg"
|
||||
)
|
||||
|
||||
# Multi-turn sequential mock client
|
||||
# Turn 1: get_ocr_list
|
||||
mock_fc_ocr = MagicMock()
|
||||
mock_fc_ocr.name = "get_ocr_list"
|
||||
mock_fc_ocr.args = {}
|
||||
mock_resp_ocr = MagicMock()
|
||||
mock_resp_ocr.function_calls = [mock_fc_ocr]
|
||||
|
||||
# Turn 2: search_xml_ocr with query
|
||||
mock_fc_search = MagicMock()
|
||||
mock_fc_search.name = "search_xml_ocr"
|
||||
mock_fc_search.args = {"query": search_query}
|
||||
mock_resp_search = MagicMock()
|
||||
mock_resp_search.function_calls = [mock_fc_search]
|
||||
|
||||
# Turn 3: detect_objects
|
||||
mock_fc_detect = MagicMock()
|
||||
mock_fc_detect.name = "detect_objects"
|
||||
mock_fc_detect.args = {"labels": labels, "target_image_id": "img_0"}
|
||||
mock_resp_detect = MagicMock()
|
||||
mock_resp_detect.function_calls = [mock_fc_detect]
|
||||
|
||||
# Turn 4: search_xml_ocr with coordinates
|
||||
mock_fc_coords = MagicMock()
|
||||
mock_fc_coords.name = "search_xml_ocr"
|
||||
mock_fc_coords.args = {"nx": coords_nx, "ny": coords_ny}
|
||||
mock_resp_coords = MagicMock()
|
||||
mock_resp_coords.function_calls = [mock_fc_coords]
|
||||
|
||||
# Turn 5: submit_answer
|
||||
mock_fc_submit = MagicMock()
|
||||
mock_fc_submit.name = "submit_answer"
|
||||
mock_fc_submit.args = {
|
||||
"candidates": candidates,
|
||||
"fallback_message": f"Successfully verified test set {set_idx}",
|
||||
}
|
||||
mock_resp_submit = MagicMock()
|
||||
mock_resp_submit.function_calls = [mock_fc_submit]
|
||||
|
||||
mock_client = MagicMock()
|
||||
|
||||
# Mock client.aio.models.generate_content side effects for ReAct turns
|
||||
mock_client.aio.models.generate_content = AsyncMock(
|
||||
side_effect=[
|
||||
mock_resp_ocr,
|
||||
mock_resp_search,
|
||||
mock_resp_detect,
|
||||
mock_resp_coords,
|
||||
mock_resp_submit,
|
||||
]
|
||||
)
|
||||
|
||||
# Mock the internal model call for _detect_single_label inside detect_objects
|
||||
mock_detect_response = MagicMock()
|
||||
y_norm, x_norm = 0, 0
|
||||
if "gear icon" in labels:
|
||||
y_norm, x_norm = 300, 200
|
||||
elif "dashboard icon" in labels:
|
||||
y_norm, x_norm = 500, 500
|
||||
elif "magnifying glass icon" in labels:
|
||||
y_norm, x_norm = 140, 875
|
||||
elif "avatar icon" in labels:
|
||||
y_norm, x_norm = 825, 375
|
||||
elif "arrow back icon" in labels:
|
||||
y_norm, x_norm = 75, 100
|
||||
|
||||
mock_detect_response.text = json.dumps([{"point": [y_norm, x_norm]}])
|
||||
|
||||
mock_detector_client = MagicMock()
|
||||
mock_detector_client.aio.models.generate_content = AsyncMock(return_value=mock_detect_response)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"google.genai.Client",
|
||||
side_effect=[mock_client, mock_detector_client],
|
||||
),
|
||||
):
|
||||
explorer = Explorer(mock_ctx)
|
||||
await explorer.run(
|
||||
query=query,
|
||||
context_feedback="",
|
||||
screenshot_path=str(screenshot_path),
|
||||
state=mock_state,
|
||||
)
|
||||
|
||||
call_args_list = mock_client.aio.models.generate_content.call_args_list
|
||||
contents = call_args_list[4].kwargs.get("contents") or call_args_list[4].args[0]
|
||||
extract_and_print_tool_results(contents, set_idx, logger)
|
||||
|
||||
|
||||
async def main():
|
||||
import shutil
|
||||
from artemis.config import settings
|
||||
|
||||
# Define outputs directory
|
||||
outputs_dir = Path(__file__).resolve().parent / "outputs"
|
||||
if outputs_dir.exists():
|
||||
shutil.rmtree(outputs_dir)
|
||||
outputs_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Redirect traces output to outputs/ and only keep the latest run
|
||||
settings.TRACES_PATH = outputs_dir
|
||||
|
||||
# 1. Seed the SQLite DB record first
|
||||
seed_database()
|
||||
|
||||
# 2. Define the 5 groups of input tests
|
||||
test_sets = [
|
||||
{
|
||||
"query": "Verify Settings button",
|
||||
"search_query": "Settings",
|
||||
"labels": ["gear icon"],
|
||||
"coords_nx": (185), # Matches pixel center [200, 225] on [1080, 2400] scale
|
||||
"coords_ny": 93,
|
||||
"candidates": [
|
||||
{
|
||||
"label": "S1",
|
||||
"coords": [185, 93],
|
||||
"description": "Settings Button",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"query": "Verify Dashboard button",
|
||||
"search_query": "Dashboard",
|
||||
"labels": ["dashboard icon"],
|
||||
"coords_nx": (462), # Matches pixel center [500, 500] on [1080, 2400] scale
|
||||
"coords_ny": 208,
|
||||
"candidates": [
|
||||
{
|
||||
"label": "S1",
|
||||
"coords": [462, 208],
|
||||
"description": "Dashboard Button",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"query": "Verify Search bar",
|
||||
"search_query": "Search",
|
||||
"labels": ["magnifying glass icon"],
|
||||
"coords_nx": (810), # Matches pixel center [875, 140] on [1080, 2400] scale
|
||||
"coords_ny": 58,
|
||||
"candidates": [
|
||||
{
|
||||
"label": "S1",
|
||||
"coords": [810, 58],
|
||||
"description": "Search bar",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"query": "Verify Profile icon",
|
||||
"search_query": "Profile",
|
||||
"labels": ["avatar icon"],
|
||||
"coords_nx": (347), # Matches pixel center [375, 825] on [1080, 2400] scale
|
||||
"coords_ny": 343,
|
||||
"candidates": [
|
||||
{
|
||||
"label": "S1",
|
||||
"coords": [347, 343],
|
||||
"description": "Profile icon",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"query": "Verify Back button",
|
||||
"search_query": "Back",
|
||||
"labels": ["arrow back icon"],
|
||||
"coords_nx": (92), # Matches pixel center [100, 75] on [1080, 2400] scale
|
||||
"coords_ny": 31,
|
||||
"candidates": [
|
||||
{
|
||||
"label": "S1",
|
||||
"coords": [92, 31],
|
||||
"description": "Back button",
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
# 3. Run all 5 sets sequentially
|
||||
for i, test in enumerate(test_sets, 1):
|
||||
await run_single_set(
|
||||
set_idx=i,
|
||||
query=test["query"],
|
||||
search_query=test["search_query"],
|
||||
labels=test["labels"],
|
||||
coords_nx=test["coords_nx"],
|
||||
coords_ny=test["coords_ny"],
|
||||
candidates=test["candidates"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,162 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
|
||||
|
||||
def seed_database():
|
||||
db_path = Path(__file__).resolve().parents[5] / "traces" / "data_engine.db"
|
||||
screenshot_path = (
|
||||
Path(__file__).resolve().parent
|
||||
/ "input_screenshot_test_explorer_all_tools_sequential_mocked.jpg"
|
||||
)
|
||||
|
||||
if not screenshot_path.exists():
|
||||
# Try fallback to tests/screenshots/pixel_phone_mockup.png
|
||||
source_path = (
|
||||
Path(__file__).resolve().parents[5] / "tests" / "screenshots" / "pixel_phone_mockup.png"
|
||||
)
|
||||
import shutil
|
||||
|
||||
shutil.copy(source_path, screenshot_path)
|
||||
print(f"Copied screenshot from {source_path} to {screenshot_path}")
|
||||
|
||||
# Compute SHA-256 hash
|
||||
sha256_hash = hashlib.sha256()
|
||||
with open(screenshot_path, "rb") as f:
|
||||
for byte_block in iter(lambda: f.read(4096), b""):
|
||||
sha256_hash.update(byte_block)
|
||||
computed_hash = sha256_hash.hexdigest()
|
||||
print(f"Computed screenshot hash: {computed_hash}")
|
||||
|
||||
# Bounding elements
|
||||
# 1. Settings [100, 200][300, 250] -> center pixel [200, 225] -> normalized [185, 93]
|
||||
# 2. Dashboard [400, 400][600, 600] -> center pixel [500, 500] -> normalized [462, 208]
|
||||
# 3. Search [800, 100][950, 180] -> center pixel [875, 140] -> normalized [810, 58]
|
||||
# 4. Profile [300, 800][450, 850] -> center pixel [375, 825] -> normalized [347, 343]
|
||||
# 5. Back [50, 50][150, 100] -> center pixel [100, 75] -> normalized [92, 31]
|
||||
|
||||
ocr_result = [
|
||||
{"text": "Screen Dashboard Title", "position": []},
|
||||
{
|
||||
"text": "Settings",
|
||||
"position": [
|
||||
{"x": 100, "y": 200},
|
||||
{"x": 300, "y": 200},
|
||||
{"x": 300, "y": 250},
|
||||
{"x": 100, "y": 250},
|
||||
],
|
||||
},
|
||||
{
|
||||
"text": "Dashboard",
|
||||
"position": [
|
||||
{"x": 400, "y": 400},
|
||||
{"x": 600, "y": 400},
|
||||
{"x": 600, "y": 600},
|
||||
{"x": 400, "y": 600},
|
||||
],
|
||||
},
|
||||
{
|
||||
"text": "Search",
|
||||
"position": [
|
||||
{"x": 800, "y": 100},
|
||||
{"x": 950, "y": 100},
|
||||
{"x": 950, "y": 180},
|
||||
{"x": 800, "y": 180},
|
||||
],
|
||||
},
|
||||
{
|
||||
"text": "Profile",
|
||||
"position": [
|
||||
{"x": 300, "y": 800},
|
||||
{"x": 450, "y": 800},
|
||||
{"x": 450, "y": 850},
|
||||
{"x": 300, "y": 850},
|
||||
],
|
||||
},
|
||||
{
|
||||
"text": "Back",
|
||||
"position": [
|
||||
{"x": 50, "y": 50},
|
||||
{"x": 150, "y": 50},
|
||||
{"x": 150, "y": 100},
|
||||
{"x": 50, "y": 100},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
ui_tree = [
|
||||
{"class": "android.widget.FrameLayout", "bounds": "[0,0][1080,2400]"},
|
||||
{
|
||||
"class": "android.widget.Button",
|
||||
"text": "Settings",
|
||||
"resource-id": "com.example.app:id/settings",
|
||||
"clickable": "true",
|
||||
"bounds": "[100,200][300,250]",
|
||||
},
|
||||
{
|
||||
"class": "android.widget.Button",
|
||||
"text": "Dashboard",
|
||||
"resource-id": "com.example.app:id/dashboard",
|
||||
"clickable": "true",
|
||||
"bounds": "[400,400][600,600]",
|
||||
},
|
||||
{
|
||||
"class": "android.widget.Button",
|
||||
"text": "Search",
|
||||
"resource-id": "com.example.app:id/search",
|
||||
"clickable": "true",
|
||||
"bounds": "[800,100][950,180]",
|
||||
},
|
||||
{
|
||||
"class": "android.widget.Button",
|
||||
"text": "Profile",
|
||||
"resource-id": "com.example.app:id/profile",
|
||||
"clickable": "true",
|
||||
"bounds": "[300,800][450,850]",
|
||||
},
|
||||
{
|
||||
"class": "android.widget.Button",
|
||||
"text": "Back",
|
||||
"resource-id": "com.example.app:id/back",
|
||||
"clickable": "true",
|
||||
"bounds": "[50,50][150,100]",
|
||||
},
|
||||
]
|
||||
|
||||
conn = sqlite3.connect(db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute(
|
||||
"INSERT OR REPLACE INTO images (image_name, timestamp, ocr_result,"
|
||||
" ui_tree, extra_metadata) VALUES (?, ?, ?, ?, ?)",
|
||||
(
|
||||
computed_hash,
|
||||
1717398000.0,
|
||||
json.dumps(ocr_result),
|
||||
json.dumps(ui_tree),
|
||||
json.dumps({}),
|
||||
),
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f"Successfully inserted mock image record {computed_hash} into database at {db_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
seed_database()
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, patch
|
||||
from artemis.agents.object_detector.object_detector import _run_object_detection
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_coordinate_normalization_swap():
|
||||
"""Verify that _run_object_detection correctly swaps coordinates from [y, x] to [x, y]."""
|
||||
|
||||
# 1. Define the mock output returned by the VLM task (VLM native format: [y_norm, x_norm])
|
||||
mock_vlm_output = [
|
||||
{
|
||||
"label": "settings icon",
|
||||
"point": [800, 200], # [y, x] -> y = 800, x = 200
|
||||
}
|
||||
]
|
||||
|
||||
# 2. Patch dependencies to run in a pure offline sandbox
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.object_detector.object_detector._detect_single_label",
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_vlm_output,
|
||||
),
|
||||
patch("artemis.agents.object_detector.object_detector.get_llm"),
|
||||
):
|
||||
# Mock context setup
|
||||
mock_ctx = AsyncMock()
|
||||
mock_ctx.llm_config.utils.object_detector = None
|
||||
|
||||
# 3. Execute the function under test
|
||||
json_output = await _run_object_detection(
|
||||
ctx=mock_ctx,
|
||||
image_bytes=b"fake_image_bytes",
|
||||
queries=["settings icon"],
|
||||
templates=["find the {label}"],
|
||||
)
|
||||
|
||||
# 4. Verify that the coordinates were successfully swapped
|
||||
result = json_output
|
||||
detected_items = result["detected"]
|
||||
|
||||
assert len(detected_items) == 1
|
||||
assert detected_items[0]["label"] == "settings icon"
|
||||
|
||||
# CRITICAL ASSERTION:
|
||||
# The coordinate MUST be swapped from [800, 200] (y, x) to [200, 800] (x, y)
|
||||
assert detected_items[0]["point"] == [200, 800]
|
||||
@@ -0,0 +1,334 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sys
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from artemis.agents.explorer.explorer import Explorer
|
||||
from tests.integration.agents.explorer.test_explorer_all_tools.helpers import (
|
||||
create_mock_context,
|
||||
create_mock_state,
|
||||
get_or_create_test_screenshot,
|
||||
)
|
||||
import pytest
|
||||
|
||||
|
||||
def _verify_all_tools_responses(contents: list, logger: logging.Logger):
|
||||
"""Parses the multi-turn conversational history payload and maps tool outputs."""
|
||||
logger.info("=== VERIFYING TOOL RESPONSES IN CONVERSATION HISTORY ===")
|
||||
tool_responses = {}
|
||||
for content in contents:
|
||||
for part in content.parts:
|
||||
if part.function_response:
|
||||
name = part.function_response.name
|
||||
response_data = part.function_response.response
|
||||
|
||||
# Extract result text
|
||||
tool_text = None
|
||||
if hasattr(response_data, "get"):
|
||||
tool_text = response_data.get("result")
|
||||
elif hasattr(response_data, "fields"):
|
||||
tool_text = response_data.fields.get("result")
|
||||
else:
|
||||
tool_text = getattr(response_data, "result", None)
|
||||
|
||||
if name not in tool_responses:
|
||||
tool_responses[name] = []
|
||||
tool_responses[name].append({"text": tool_text, "has_image": False})
|
||||
logger.info(f"Found tool response for '{name}':\n{tool_text}")
|
||||
|
||||
if part.inline_data and part.inline_data.mime_type == "image/jpeg":
|
||||
# The image byte block corresponds to the preceding function response
|
||||
if tool_responses:
|
||||
last_tool = list(tool_responses.keys())[-1]
|
||||
tool_responses[last_tool][-1]["has_image"] = True
|
||||
tool_responses[last_tool][-1]["img_bytes"] = part.inline_data.data
|
||||
logger.info(
|
||||
f" -> Attached {len(part.inline_data.data)} bytes of"
|
||||
f" JPEG inline visual content to '{last_tool}'"
|
||||
)
|
||||
|
||||
return tool_responses
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_explorer_all_tools_sequential_mocked():
|
||||
"""Verify that the Explorer ReAct loop correctly sequences, dispatches, processes, and
|
||||
|
||||
annotates screenshot outputs for all available visual tools:
|
||||
1. get_ocr_list
|
||||
2. search_xml_ocr (query)
|
||||
3. detect_objects
|
||||
4. search_xml_ocr (coordinates)
|
||||
5. submit_answer
|
||||
"""
|
||||
test_name = sys._getframe().f_code.co_name
|
||||
|
||||
# Set up standard log file saved on the same level as test file
|
||||
log_file = Path(__file__).parent / f"log_{test_name}.log"
|
||||
logger = logging.getLogger(test_name)
|
||||
logger.setLevel(logging.INFO)
|
||||
handler = logging.FileHandler(log_file, mode="w", encoding="utf-8")
|
||||
logger.addHandler(handler)
|
||||
|
||||
# Attach the main agent logger to capture internal step logs
|
||||
agent_logger = logging.getLogger("artemis.agents.explorer.explorer")
|
||||
agent_logger.setLevel(logging.INFO)
|
||||
agent_logger.addHandler(handler)
|
||||
|
||||
logger.info(f"Starting comprehensive test: {test_name}")
|
||||
|
||||
# Define outputs directory and redirect settings to ensure only the latest run is saved
|
||||
outputs_dir = Path(__file__).resolve().parent / "outputs"
|
||||
if outputs_dir.exists():
|
||||
shutil.rmtree(outputs_dir)
|
||||
outputs_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
from artemis.config import settings
|
||||
|
||||
settings.TRACES_PATH = outputs_dir
|
||||
|
||||
mock_ctx = create_mock_context()
|
||||
mock_state = create_mock_state()
|
||||
|
||||
logger.info(
|
||||
f"Mock context: device={mock_ctx.device.device_width}x{mock_ctx.device.device_height}"
|
||||
)
|
||||
|
||||
# Load and verify screenshot
|
||||
input_screenshot = get_or_create_test_screenshot()
|
||||
logger.info(f"Visual screenshot loaded: {input_screenshot}")
|
||||
|
||||
# =========================================================================
|
||||
# 1. Mock LLM Multi-Turn Responses (Sequentially calling each tool)
|
||||
# =========================================================================
|
||||
|
||||
# Turn 1: call get_ocr_list
|
||||
mock_fc_ocr = MagicMock()
|
||||
mock_fc_ocr.name = "get_ocr_list"
|
||||
mock_fc_ocr.args = {}
|
||||
mock_resp_ocr = MagicMock()
|
||||
mock_resp_ocr.function_calls = [mock_fc_ocr]
|
||||
|
||||
# Turn 2: call ask_perception_tool
|
||||
mock_fc_search = MagicMock()
|
||||
mock_fc_search.name = "ask_perception_tool"
|
||||
mock_fc_search.args = {
|
||||
"search_query": "Settings",
|
||||
"nx": 500,
|
||||
"ny": 500,
|
||||
"detect_queries": ["gear icon"],
|
||||
}
|
||||
mock_resp_search = MagicMock()
|
||||
mock_resp_search.function_calls = [mock_fc_search]
|
||||
|
||||
# Turn 3: call detect_objects
|
||||
mock_fc_detect = MagicMock()
|
||||
mock_fc_detect.name = "detect_objects"
|
||||
mock_fc_detect.args = {"queries": ["gear icon"], "target_image_id": "img_0"}
|
||||
mock_resp_detect = MagicMock()
|
||||
mock_resp_detect.function_calls = [mock_fc_detect]
|
||||
|
||||
# Turn 4: call inspect_region
|
||||
mock_fc_coords = MagicMock()
|
||||
mock_fc_coords.name = "inspect_region"
|
||||
mock_fc_coords.args = {
|
||||
"x_min": 100,
|
||||
"y_min": 100,
|
||||
"x_max": 300,
|
||||
"y_max": 300,
|
||||
"zoom_factor": 2.0,
|
||||
}
|
||||
mock_resp_coords = MagicMock()
|
||||
mock_resp_coords.function_calls = [mock_fc_coords]
|
||||
|
||||
# Turn 5: call submit_answer to complete loop
|
||||
mock_fc_submit = MagicMock()
|
||||
mock_fc_submit.name = "submit_answer"
|
||||
mock_fc_submit.args = {
|
||||
"candidates": [
|
||||
{
|
||||
"label": "T1",
|
||||
"coords": [462, 208],
|
||||
"description": "Dashboard Button",
|
||||
},
|
||||
{
|
||||
"label": "S1",
|
||||
"coords": [462, 208],
|
||||
"description": "Settings Button",
|
||||
},
|
||||
{"label": "D1", "coords": [200, 300], "description": "gear icon"},
|
||||
],
|
||||
"fallback_message": ("Successfully verified all visual controls sequentially."),
|
||||
}
|
||||
mock_resp_submit = MagicMock()
|
||||
mock_resp_submit.function_calls = [mock_fc_submit]
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.aio.models.generate_content = AsyncMock(
|
||||
side_effect=[
|
||||
mock_resp_ocr,
|
||||
mock_resp_search,
|
||||
mock_resp_detect,
|
||||
mock_resp_coords,
|
||||
mock_resp_submit,
|
||||
]
|
||||
)
|
||||
|
||||
mock_ctx._genai_client = mock_client
|
||||
mock_ctx.llm_config.explorer = None
|
||||
|
||||
# =========================================================================
|
||||
# 2. Mock Downstream Tool Logic Dependencies
|
||||
# =========================================================================
|
||||
|
||||
# Mock SQLite StorageManager for get_ocr_list
|
||||
mock_storage = MagicMock()
|
||||
mock_record = MagicMock()
|
||||
mock_record.ocr_result = [
|
||||
{"text": "screen description text", "position": []},
|
||||
{
|
||||
"text": "Dashboard Button",
|
||||
"position": [
|
||||
{"x": 400, "y": 400},
|
||||
{"x": 600, "y": 400},
|
||||
{"x": 600, "y": 600},
|
||||
{"x": 400, "y": 600},
|
||||
],
|
||||
},
|
||||
]
|
||||
mock_storage.get_image.return_value = mock_record
|
||||
|
||||
# Mock low-level layout searching (search_ui_func)
|
||||
mock_search_ui = MagicMock(
|
||||
return_value={
|
||||
"matches": [
|
||||
{
|
||||
"matched_text": "Settings Button",
|
||||
"bounds": [400, 400, 600, 600],
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
# Mock downstream object detection pipeline (_run_object_detection)
|
||||
mock_detect = AsyncMock(
|
||||
return_value={"detected": [{"label": "gear icon", "point": [200, 300]}]}
|
||||
)
|
||||
|
||||
# Mock spatial coordinate lookup (search_by_coordinates_func)
|
||||
mock_lookup = MagicMock(return_value="Matching Elements: Container/DashboardFrame")
|
||||
|
||||
logger.info("Starting Explorer.run execution loop with sequential tool mocks...")
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.explorer.explorer.genai.Client",
|
||||
return_value=mock_client,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.explorer.explorer.StorageManager",
|
||||
return_value=mock_storage,
|
||||
),
|
||||
patch("artemis.agents.explorer.explorer.search_ui_func", mock_search_ui),
|
||||
patch("artemis.agents.explorer.explorer._run_object_detection", mock_detect),
|
||||
patch(
|
||||
"artemis.agents.explorer.explorer.search_by_coordinates_func",
|
||||
mock_lookup,
|
||||
),
|
||||
):
|
||||
explorer = Explorer(mock_ctx)
|
||||
result = await explorer.run(
|
||||
query="Test all tools",
|
||||
context_feedback="",
|
||||
screenshot_path=str(input_screenshot),
|
||||
state=mock_state,
|
||||
version="ultra",
|
||||
)
|
||||
|
||||
# Verify ReAct loop went through all 5 turns
|
||||
assert mock_client.aio.models.generate_content.call_count == 5
|
||||
logger.info("Successfully completed all 5 ReAct turns.")
|
||||
|
||||
# Retrieve final submission outcome
|
||||
outcome_data = json.loads(result)
|
||||
assert "candidates" in outcome_data
|
||||
assert (
|
||||
outcome_data["fallback_message"]
|
||||
== "Successfully verified all visual controls sequentially."
|
||||
)
|
||||
logger.info("Submission response verified successfully.")
|
||||
|
||||
# Extract all tool outputs from history to assert correctness
|
||||
call_args_list = mock_client.aio.models.generate_content.call_args_list
|
||||
# The fifth call receives the accumulated tool history
|
||||
contents = call_args_list[4].kwargs.get("contents") or call_args_list[4].args[0]
|
||||
|
||||
tool_responses = _verify_all_tools_responses(contents, logger)
|
||||
|
||||
# ASSERTIONS FOR EACH TOOL INDIVIDUALLY
|
||||
|
||||
# Verify each tool execution record in explorer.trace_history
|
||||
assert len(explorer.trace_history) >= 4
|
||||
# 1. Verify get_ocr_list
|
||||
ocr_trace = explorer.trace_history[0]["tool_calls"][0]
|
||||
assert ocr_trace["name"] == "get_ocr_list"
|
||||
assert "[T1]" in ocr_trace["response"]["text"]
|
||||
assert "[462,208]" in ocr_trace["response"]["text"]
|
||||
assert ocr_trace["response"]["image_path"] is not None
|
||||
|
||||
# 2. Verify ask_perception_tool
|
||||
search_trace = explorer.trace_history[1]["tool_calls"][0]
|
||||
assert search_trace["name"] == "ask_perception_tool"
|
||||
assert "XML/OCR Text Search Results" in search_trace["response"]["text"]
|
||||
assert len(search_trace["response"]["image_paths"]) > 0
|
||||
|
||||
# 3. Verify detect_objects
|
||||
detect_trace = explorer.trace_history[2]["tool_calls"][0]
|
||||
assert detect_trace["name"] == "detect_objects"
|
||||
assert "[D" in detect_trace["response"]["text"]
|
||||
assert detect_trace["response"]["image_path"] is not None
|
||||
|
||||
# 4. Verify inspect_region
|
||||
coords_trace = explorer.trace_history[3]["tool_calls"][0]
|
||||
assert coords_trace["name"] == "inspect_region"
|
||||
assert coords_trace["response"]["image_path"] is not None
|
||||
# Compute expected screenshot hash
|
||||
import hashlib
|
||||
|
||||
sha256_hash = hashlib.sha256()
|
||||
with open(input_screenshot, "rb") as f:
|
||||
for byte_block in iter(lambda: f.read(4096), b""):
|
||||
sha256_hash.update(byte_block)
|
||||
expected_hash = sha256_hash.hexdigest()
|
||||
|
||||
# Should map normalized [500, 500] to pixel [540, 1200]
|
||||
mock_lookup.assert_called_once_with(expected_hash, 540, 1200)
|
||||
|
||||
logger.info(
|
||||
"All assertions passed! The entire suite of explorer tools is fully functional."
|
||||
)
|
||||
|
||||
# Clean up logging handlers
|
||||
agent_logger.removeHandler(handler)
|
||||
logger.removeHandler(handler)
|
||||
handler.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_explorer_all_tools_sequential_mocked())
|
||||
+372
@@ -0,0 +1,372 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from artemis.agents.explorer.explorer import Explorer
|
||||
from tests.integration.agents.explorer.test_explorer_all_tools.helpers import (
|
||||
create_mock_context,
|
||||
create_mock_state,
|
||||
get_or_create_test_screenshot,
|
||||
)
|
||||
from PIL import Image
|
||||
import pytest
|
||||
|
||||
|
||||
def _verify_tool_responses(contents: list, logger: logging.Logger):
|
||||
"""Parses the multi-turn conversational history payload and maps tool outputs."""
|
||||
logger.info("=== VERIFYING TOOL RESPONSES IN CONVERSATION HISTORY ===")
|
||||
tool_responses = {}
|
||||
for content in contents:
|
||||
for part in content.parts:
|
||||
if part.function_response:
|
||||
name = part.function_response.name
|
||||
response_data = part.function_response.response
|
||||
|
||||
# Extract result text
|
||||
tool_text = None
|
||||
if hasattr(response_data, "get"):
|
||||
tool_text = response_data.get("result")
|
||||
elif hasattr(response_data, "fields"):
|
||||
tool_text = response_data.fields.get("result")
|
||||
else:
|
||||
tool_text = getattr(response_data, "result", None)
|
||||
|
||||
if name not in tool_responses:
|
||||
tool_responses[name] = []
|
||||
tool_responses[name].append({"text": tool_text, "has_image": False})
|
||||
logger.info(f"Found tool response for '{name}':\n{tool_text}")
|
||||
|
||||
if part.inline_data and part.inline_data.mime_type == "image/jpeg":
|
||||
if tool_responses:
|
||||
last_tool = list(tool_responses.keys())[-1]
|
||||
tool_responses[last_tool][-1]["has_image"] = True
|
||||
tool_responses[last_tool][-1]["img_bytes"] = part.inline_data.data
|
||||
logger.info(
|
||||
f" -> Attached {len(part.inline_data.data)} bytes of"
|
||||
f" JPEG inline visual content to '{last_tool}'"
|
||||
)
|
||||
|
||||
return tool_responses
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_explorer_ask_image_processor_tool():
|
||||
"""Verify that the Explorer ReAct loop correctly dispatches, runs, processes, and
|
||||
|
||||
handles the output image/transform details for the 'ask_image_processor'
|
||||
tool.
|
||||
The test ends immediately after the Vision Coder responds (in the second
|
||||
turn).
|
||||
"""
|
||||
test_name = sys._getframe().f_code.co_name
|
||||
|
||||
# Set up standard log file saved on the same level as test file
|
||||
log_file = Path(__file__).parent / f"log_{test_name}.log"
|
||||
logger = logging.getLogger(test_name)
|
||||
logger.setLevel(logging.INFO)
|
||||
handler = logging.FileHandler(log_file, mode="w", encoding="utf-8")
|
||||
logger.addHandler(handler)
|
||||
|
||||
# Attach the main agent logger to capture internal step logs
|
||||
agent_logger = logging.getLogger("artemis.agents.explorer.explorer")
|
||||
agent_logger.setLevel(logging.INFO)
|
||||
agent_logger.addHandler(handler)
|
||||
|
||||
logger.info(f"Starting test: {test_name}")
|
||||
|
||||
# Define outputs directory and redirect settings to ensure only the latest run is saved
|
||||
outputs_dir = Path(__file__).resolve().parent / "outputs"
|
||||
if not outputs_dir.exists():
|
||||
outputs_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
from artemis.config import settings
|
||||
|
||||
settings.TRACES_PATH = outputs_dir
|
||||
|
||||
mock_ctx = create_mock_context()
|
||||
mock_state = create_mock_state()
|
||||
|
||||
logger.info(
|
||||
f"Mock context: device={mock_ctx.device.device_width}x{mock_ctx.device.device_height}"
|
||||
)
|
||||
|
||||
# Load and verify screenshot
|
||||
input_screenshot = get_or_create_test_screenshot()
|
||||
logger.info(f"Visual screenshot loaded: {input_screenshot}")
|
||||
|
||||
# =========================================================================
|
||||
# 1. Mock LLM Multi-Turn Responses (ask_image_processor -> submit_answer)
|
||||
# =========================================================================
|
||||
|
||||
# Turn 1: call ask_image_processor
|
||||
mock_fc_coder = MagicMock()
|
||||
mock_fc_coder.name = "ask_image_processor"
|
||||
mock_fc_coder.args = {
|
||||
"instruction": (
|
||||
"Crop the top-right area, convert to HSV, resize, then convert to grayscale."
|
||||
),
|
||||
"target_image_id": "img_0",
|
||||
}
|
||||
mock_resp_coder = MagicMock()
|
||||
mock_resp_coder.function_calls = [mock_fc_coder]
|
||||
|
||||
# Turn 2: call submit_answer to complete loop immediately after vision coder responds
|
||||
mock_fc_submit = MagicMock()
|
||||
mock_fc_submit.name = "submit_answer"
|
||||
mock_fc_submit.args = {
|
||||
"candidates": [],
|
||||
"fallback_message": ("Successfully processed the image via Vision Coder."),
|
||||
}
|
||||
mock_resp_submit = MagicMock()
|
||||
mock_resp_submit.function_calls = [mock_fc_submit]
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.aio.models.generate_content = AsyncMock(
|
||||
side_effect=[mock_resp_coder, mock_resp_submit]
|
||||
)
|
||||
|
||||
mock_ctx._genai_client = mock_client
|
||||
mock_ctx.llm_config.explorer = None
|
||||
|
||||
# =========================================================================
|
||||
# 2. Mock ImageProcessor agent
|
||||
# =========================================================================
|
||||
|
||||
# Create the mock output image path
|
||||
mock_new_image_path = outputs_dir / f"mock_image_processor_output_{test_name}.jpg"
|
||||
# Create a small dummy image of size 100x100 using PIL so that cv2.imread inside explorer reads these dimensions
|
||||
img = Image.new("RGB", (100, 100), color="red")
|
||||
img.save(mock_new_image_path)
|
||||
|
||||
# Also write a transform JSON file for the new image
|
||||
transform_path = Path(str(mock_new_image_path) + ".transform.json")
|
||||
mock_transform_data = {
|
||||
"offset_x": 108.0,
|
||||
"offset_y": 240.0,
|
||||
"scale_x": 0.5,
|
||||
"scale_y": 0.5,
|
||||
}
|
||||
with open(transform_path, "w") as f:
|
||||
json.dump(mock_transform_data, f)
|
||||
|
||||
# We mock ImageProcessor class itself
|
||||
mock_image_processor_instance = MagicMock()
|
||||
mock_image_processor_instance.run = AsyncMock(
|
||||
return_value={
|
||||
"outputs": [
|
||||
{
|
||||
"image_id": "img_1",
|
||||
"path": str(mock_new_image_path),
|
||||
"transform": {
|
||||
"offset_x": 108.0,
|
||||
"offset_y": 240.0,
|
||||
"scale_x": 0.5,
|
||||
"scale_y": 0.5,
|
||||
},
|
||||
"annotations": {"V1": [50, 100], "V2": [180, 240]},
|
||||
}
|
||||
],
|
||||
"summary": "Cropped and highlighted the red button",
|
||||
}
|
||||
)
|
||||
|
||||
logger.info("Starting Explorer.run execution loop with ask_image_processor...")
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.explorer.explorer.genai.Client",
|
||||
return_value=mock_client,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.image_processor.image_processor.ImageProcessor",
|
||||
return_value=mock_image_processor_instance,
|
||||
),
|
||||
):
|
||||
explorer = Explorer(mock_ctx)
|
||||
|
||||
result = await explorer.run(
|
||||
query=("Click on the red button in the top-right corner of the screen."),
|
||||
context_feedback="",
|
||||
screenshot_path=str(input_screenshot),
|
||||
state=mock_state,
|
||||
version="ultra",
|
||||
)
|
||||
|
||||
# Verify initial image pool has img_0
|
||||
assert "img_0" in explorer.image_pool
|
||||
assert explorer.image_pool["img_0"]["path"] == str(input_screenshot)
|
||||
|
||||
# Verify ReAct loop went through exactly 2 turns
|
||||
assert mock_client.aio.models.generate_content.call_count == 2
|
||||
logger.info("Successfully completed both ReAct turns.")
|
||||
|
||||
# Verify ImageProcessor run was called with correct parameters
|
||||
mock_image_processor_instance.run.assert_called_once_with(
|
||||
"Crop the top-right area, convert to HSV, resize, then convert to grayscale.",
|
||||
str(input_screenshot),
|
||||
)
|
||||
logger.info("Verified ImageProcessor was invoked with correct parameters.")
|
||||
|
||||
# Verify new image is registered in explorer's image_pool
|
||||
assert "img_1" in explorer.image_pool
|
||||
img_1_info = explorer.image_pool["img_1"]
|
||||
assert img_1_info["path"] == str(mock_new_image_path)
|
||||
|
||||
# Verify transform calculations:
|
||||
# Since parent (img_0) transform is {"offset_x": 0.0, "offset_y": 0.0, "scale_x": 1.0, "scale_y": 1.0}
|
||||
# and child transform is {"offset_x": 108.0, "offset_y": 240.0, "scale_x": 0.5, "scale_y": 0.5}
|
||||
assert img_1_info["transform"]["scale_x"] == 0.5
|
||||
assert img_1_info["transform"]["scale_y"] == 0.5
|
||||
assert img_1_info["transform"]["offset_x"] == 108.0
|
||||
assert img_1_info["transform"]["offset_y"] == 240.0
|
||||
assert "Cropped and highlighted the red button" in img_1_info["description"]
|
||||
logger.info("Verified image_pool registrations and coordinate transforms.")
|
||||
|
||||
# Retrieve final submission outcome
|
||||
outcome_data = json.loads(result)
|
||||
assert (
|
||||
outcome_data["fallback_message"] == "Successfully processed the image via Vision Coder."
|
||||
)
|
||||
logger.info("Submission response verified successfully.")
|
||||
|
||||
# Extract all tool outputs from history to assert correctness
|
||||
call_args_list = mock_client.aio.models.generate_content.call_args_list
|
||||
# The second call receives the accumulated tool history
|
||||
contents = call_args_list[1].kwargs.get("contents") or call_args_list[1].args[0]
|
||||
|
||||
tool_responses = _verify_tool_responses(contents, logger)
|
||||
|
||||
# Assertions for ask_image_processor tool response (Turn 1 output)
|
||||
assert "ask_image_processor" in tool_responses
|
||||
coder_resp = tool_responses["ask_image_processor"][0]
|
||||
assert "Image Processor completed successfully" in coder_resp["text"]
|
||||
assert "img_1" in coder_resp["text"]
|
||||
assert "Annotations (wrt original screenshot):" in coder_resp["text"]
|
||||
assert "img_1:" in coder_resp["text"]
|
||||
assert "- [V1]: [208, 440]" in coder_resp["text"]
|
||||
assert "- [V2]: [468, 720]" in coder_resp["text"]
|
||||
assert coder_resp["has_image"]
|
||||
assert len(coder_resp["img_bytes"]) > 0
|
||||
logger.info(
|
||||
"Verified tool response contains the response text, annotations"
|
||||
" list, and the attached image bytes."
|
||||
)
|
||||
|
||||
# Clean up dummy files (Commented out temporarily so you can see the images!)
|
||||
# if mock_new_image_path.exists():
|
||||
# mock_new_image_path.unlink()
|
||||
# if transform_path.exists():
|
||||
# transform_path.unlink()
|
||||
# if input_screenshot.exists():
|
||||
# input_screenshot.unlink()
|
||||
|
||||
# Clean up logging handlers
|
||||
agent_logger.removeHandler(handler)
|
||||
logger.removeHandler(handler)
|
||||
handler.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_explorer_ask_image_processor_tool())
|
||||
|
||||
|
||||
def test_cv_canvas_annotations(tmp_path):
|
||||
import numpy as np
|
||||
import cv2
|
||||
from artemis.utils.cv_canvas import ImageCanvas
|
||||
|
||||
# Create a dummy image
|
||||
img_dir = tmp_path / "images"
|
||||
img_dir.mkdir()
|
||||
img_path = img_dir / "img_0.jpg"
|
||||
img_data = np.zeros((100, 100, 3), dtype=np.uint8)
|
||||
cv2.imwrite(str(img_path), img_data)
|
||||
|
||||
# Initialize transforms registry
|
||||
registry_path = img_dir / "intermediate_transforms.json"
|
||||
registry_data = {
|
||||
"img_0": {
|
||||
"image_id": "img_0",
|
||||
"path": str(img_path.resolve()),
|
||||
"transform": {
|
||||
"offset_x": 0.0,
|
||||
"offset_y": 0.0,
|
||||
"scale_x": 1.0,
|
||||
"scale_y": 1.0,
|
||||
},
|
||||
"annotations": {},
|
||||
}
|
||||
}
|
||||
with open(registry_path, "w") as f:
|
||||
json.dump(registry_data, f)
|
||||
|
||||
canvas = ImageCanvas("img_0", str(img_dir))
|
||||
assert canvas.annotations == {}
|
||||
|
||||
# Test _get_resolution and initial width/height
|
||||
assert canvas._get_resolution() == (100, 100)
|
||||
assert canvas.width == 100
|
||||
assert canvas.height == 100
|
||||
|
||||
# Test normalized_to_pixel_coords
|
||||
assert canvas.normalized_to_pixel_coords(500, 500) == (50, 50)
|
||||
assert canvas.normalized_to_pixel_coords(0, 0) == (0, 0)
|
||||
assert canvas.normalized_to_pixel_coords(1000, 1000) == (99, 99)
|
||||
|
||||
# Draw dot
|
||||
canvas.draw_dot(10, 20, 1)
|
||||
assert canvas.annotations == {"V1": [10, 20]}
|
||||
|
||||
# Crop
|
||||
canvas.crop(5, 5, 20, 20)
|
||||
assert canvas.width == 20
|
||||
assert canvas.height == 20
|
||||
assert canvas._get_resolution() == (20, 20)
|
||||
assert canvas.normalized_to_pixel_coords(500, 500) == (10, 10)
|
||||
assert canvas.annotations == {"V1": [5, 15]}
|
||||
|
||||
# Out of bounds test
|
||||
canvas.draw_dot(2, 2, 2)
|
||||
assert canvas.annotations == {"V1": [5, 15], "V2": [2, 2]}
|
||||
|
||||
# Crop again to discard V1
|
||||
canvas.crop(1, 1, 5, 5)
|
||||
assert canvas.width == 5
|
||||
assert canvas.height == 5
|
||||
assert canvas.normalized_to_pixel_coords(500, 500) == (2, 2)
|
||||
# V1 was at [5, 15], after crop(1, 1), it becomes [4, 14], which is out of bounds of 5x5.
|
||||
# V2 was at [2, 2], after crop(1, 1), it becomes [1, 1], which is inside 5x5.
|
||||
assert canvas.annotations == {"V2": [1, 1]}
|
||||
|
||||
# Resize by factor
|
||||
canvas.resize_by_factor(2.0)
|
||||
assert canvas.width == 10
|
||||
assert canvas.height == 10
|
||||
assert canvas._get_resolution() == (10, 10)
|
||||
assert canvas.normalized_to_pixel_coords(500, 500) == (5, 5)
|
||||
assert canvas.annotations == {"V2": [2, 2]}
|
||||
|
||||
# Save and verify registry
|
||||
canvas.save(final=True)
|
||||
with open(registry_path) as f:
|
||||
saved_registry = json.load(f)
|
||||
|
||||
assert "img_1" in saved_registry
|
||||
assert saved_registry["img_1"]["annotations"] == {"V2": [2, 2]}
|
||||
assert saved_registry["img_1"]["is_output"] is True
|
||||
+232
@@ -0,0 +1,232 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
from unittest.mock import patch
|
||||
|
||||
from artemis.config import settings
|
||||
from artemis.agents.image_processor.image_processor import ImageProcessor
|
||||
from tests.integration.agents.explorer.test_explorer_all_tools.helpers import (
|
||||
create_mock_context,
|
||||
get_or_create_test_screenshot,
|
||||
)
|
||||
from artemis.utils.python_executor import PythonExecutor
|
||||
import pytest
|
||||
|
||||
# Setup logging
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger("test_explorer_ask_image_processor_jupyter")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_explorer_ask_image_processor_jupyter():
|
||||
test_name = "test_explorer_ask_image_processor_jupyter"
|
||||
|
||||
# 1. Setup outputs directory
|
||||
outputs_dir = Path(__file__).resolve().parent / "outputs" / test_name
|
||||
outputs_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Redirect traces path to the new outputs directory
|
||||
settings.TRACES_PATH = outputs_dir
|
||||
|
||||
# 2. Prepare the input image
|
||||
src_image_path = get_or_create_test_screenshot()
|
||||
input_screenshot = outputs_dir / "input_screenshot.jpg"
|
||||
shutil.copy2(src_image_path, input_screenshot)
|
||||
logger.info(f"Copied input screenshot to: {input_screenshot}")
|
||||
|
||||
# 3. Formulate the instruction with the injected G-Logo coordinate prior
|
||||
instruction = (
|
||||
'The user wants to find "the red patch above the green patch". Let\'s'
|
||||
" analyze the screen layout. \nIn the dock / app icons, we have:\n-"
|
||||
" Play Store (9): colorful triangle (has green, blue, yellow, red)\n-"
|
||||
" Gmail (10): M icon (red, yellow, green, blue)\n- Photos (11):"
|
||||
" pinwheel (red, yellow, green, blue)\n- Chrome (16): circular logo"
|
||||
" with green on the bottom, red on top, yellow on the side, blue"
|
||||
" center.\nWait, let's look at the Photos icon (11) or Chrome icon (16)"
|
||||
" or Gmail (10) or Google search bar logo (21) or Play Store"
|
||||
" (9).\nLet's zoom into Chrome (16), Gmail (10), Photos (11), Play"
|
||||
" Store (9), Google logo (21) to analyze the relative positions of red"
|
||||
" patches and green patches.\nThe G-Logo is exactly at pixel"
|
||||
" coordinates [2173, 107, 2241, 174].\nLet's crop these icons to find"
|
||||
' which icon has a "red patch above the green patch" or is it a'
|
||||
" specific patch?\nWait!\nLet's look at Chrome (16):\nRed is at the"
|
||||
" top, green is at the bottom/left. Red is directly above green? Yes,"
|
||||
" the top slice of Chrome is red, and the bottom slice is green.\nLet's"
|
||||
" look at Photos (11):\nIt has four petals. The top petal is red? No,"
|
||||
" top petal is red/pink, bottom is green? Let's check.\nLet's look at"
|
||||
" Gmail (10):\nThe red is the M shape, green is at the bottom"
|
||||
" right?\nLet's write Python code to crop these areas and output the"
|
||||
" crop to see clearly.\nSpecifically, let's crop:\n- Play Store (bounds"
|
||||
" around 9)\n- Gmail (bounds around 10)\n- Photos (bounds around 11)\n-"
|
||||
" Chrome (bounds around 16)\n- Google widget logo (bounds around"
|
||||
" 21)\nWe will save a visualization of these cropped regions side by"
|
||||
" side or separately so we can examine the colors."
|
||||
)
|
||||
|
||||
# 4. Intercept Python executions to record them for the Jupyter Notebook
|
||||
executed_turns = []
|
||||
original_execute = PythonExecutor.execute
|
||||
|
||||
def wrapped_execute(self, code: str) -> str:
|
||||
logger.info(f"Intercepting Python code execution. Code:\n{code}\n")
|
||||
result = original_execute(self, code)
|
||||
logger.info(f"Execution result:\n{result}\n")
|
||||
executed_turns.append({"code": code, "output": result})
|
||||
return result
|
||||
|
||||
from langchain_core.messages import AIMessage
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
sample_code = """
|
||||
from artemis.utils.cv_canvas import ImageCanvas
|
||||
import glob
|
||||
from pathlib import Path
|
||||
from artemis.config import settings
|
||||
|
||||
dirs = sorted(glob.glob(str(Path(settings.TRACES_PATH) / "images" / "image_processor" / "intermediates_*")))
|
||||
intermediate_dir = dirs[-1] if dirs else "."
|
||||
|
||||
canvas = ImageCanvas("img_0", intermediate_dir)
|
||||
canvas.crop(10, 10, 50, 50)
|
||||
canvas.save(final=True)
|
||||
"""
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_bound = MagicMock()
|
||||
mock_bound.ainvoke = AsyncMock(
|
||||
side_effect=[
|
||||
AIMessage(
|
||||
content="",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "execute_python",
|
||||
"args": {"code": sample_code},
|
||||
"id": "call_1",
|
||||
}
|
||||
],
|
||||
),
|
||||
AIMessage(
|
||||
content="",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "submit_result",
|
||||
"args": {"summary": "Cropped target G-Logo region."},
|
||||
"id": "call_2",
|
||||
}
|
||||
],
|
||||
),
|
||||
]
|
||||
)
|
||||
mock_llm.bind_tools.return_value = mock_bound
|
||||
|
||||
mock_ctx = create_mock_context()
|
||||
image_processor = ImageProcessor(mock_ctx)
|
||||
|
||||
# Run with patched executor and mock LLM
|
||||
logger.info("Running Image Processor with G-Logo prior...")
|
||||
with (
|
||||
patch.object(PythonExecutor, "execute", wrapped_execute),
|
||||
patch(
|
||||
"artemis.agents.image_processor.image_processor.get_llm",
|
||||
return_value=mock_llm,
|
||||
),
|
||||
):
|
||||
result = await image_processor.run(instruction, str(input_screenshot))
|
||||
|
||||
logger.info(f"Image Processor finished. Result: {result}")
|
||||
|
||||
# 5. Compile and save the Jupyter Notebook (.ipynb)
|
||||
notebook_cells = []
|
||||
# Add a markdown introduction cell
|
||||
notebook_cells.append(
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Image Processor Execution Trace\n",
|
||||
f"**Task**: {instruction[:150]}...\n",
|
||||
"**Input Image**: `input_screenshot.jpg`",
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
for turn_idx, turn in enumerate(executed_turns):
|
||||
# Add code cell
|
||||
notebook_cells.append(
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": turn_idx + 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [line + "\n" for line in turn["output"].split("\n")],
|
||||
}
|
||||
],
|
||||
"source": [line + "\n" for line in turn["code"].split("\n")],
|
||||
}
|
||||
)
|
||||
|
||||
# If a result was submitted, add a final markdown cell
|
||||
outputs = result.get("outputs", [])
|
||||
if outputs:
|
||||
final_image_path = outputs[-1]["path"]
|
||||
notebook_cells.append(
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# 🎉 Execution Completed Successfully\n",
|
||||
f"**Summary**: {result.get('summary')}\n",
|
||||
f"**Output Image**: `{Path(final_image_path).name}`",
|
||||
],
|
||||
}
|
||||
)
|
||||
# Copy final image to output directory root for easy access
|
||||
shutil.copy2(final_image_path, outputs_dir / "final_output.jpg")
|
||||
|
||||
notebook_json = {
|
||||
"cells": notebook_cells,
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3",
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2,
|
||||
}
|
||||
|
||||
notebook_path = outputs_dir / "vision_coder_execution.ipynb"
|
||||
with open(notebook_path, "w", encoding="utf-8") as f:
|
||||
json.dump(notebook_json, f, indent=2)
|
||||
|
||||
logger.info(f"Saved compiled Jupyter notebook to: {notebook_path}")
|
||||
|
||||
# 6. Assertions
|
||||
assert "error" not in result, f"Image Processor failed: {result.get('error')}"
|
||||
assert "outputs" in result, "Image Processor did not return outputs"
|
||||
assert len(result["outputs"]) > 0, "Image Processor did not return any output images"
|
||||
logger.info("Test completed successfully! All assertions passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_explorer_ask_image_processor_jupyter())
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.agents.object_detector.object_detector import _run_object_detection
|
||||
from tests.integration.agents.explorer.test_explorer_all_tools.helpers import create_mock_context
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_object_detector_concurrency_and_throttling(tmp_path):
|
||||
"""Verify that object detector requests run in parallel and are correctly throttled by the semaphore."""
|
||||
# Create a dummy image file for the test
|
||||
dummy_image_path = tmp_path / "dummy.jpg"
|
||||
dummy_image_path.write_bytes(b"dummy image data")
|
||||
|
||||
# Setup mock context
|
||||
mock_ctx = create_mock_context()
|
||||
|
||||
# Configure the object detector LLM config explicitly to avoid MagicMocks for timeout and model
|
||||
od_cfg = MagicMock()
|
||||
od_cfg.model = "gemini-robotics-er-early-access"
|
||||
od_cfg.fix_model = "gemini-2.5-flash"
|
||||
od_cfg.timeout = 10.0
|
||||
mock_ctx.llm_config.utils.object_detector = od_cfg
|
||||
|
||||
# Track active requests
|
||||
active_requests = 0
|
||||
max_concurrent_requests = 0
|
||||
request_lock = asyncio.Lock()
|
||||
from langchain_core.messages import AIMessage
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_llm.ainvoke = AsyncMock(side_effect=lambda *args, **kwargs: mock_generate_content())
|
||||
|
||||
async def mock_generate_content(*args, **kwargs):
|
||||
nonlocal active_requests, max_concurrent_requests
|
||||
async with request_lock:
|
||||
active_requests += 1
|
||||
if active_requests > max_concurrent_requests:
|
||||
max_concurrent_requests = active_requests
|
||||
|
||||
try:
|
||||
await asyncio.sleep(0.1)
|
||||
finally:
|
||||
async with request_lock:
|
||||
active_requests -= 1
|
||||
|
||||
return AIMessage(content='[{"point": [500, 500]}]')
|
||||
|
||||
mock_llm.ainvoke = AsyncMock(side_effect=mock_generate_content)
|
||||
|
||||
with patch(
|
||||
"artemis.agents.object_detector.object_detector.get_llm",
|
||||
return_value=mock_llm,
|
||||
):
|
||||
# --- TEST 1: Concurrency Assertion (5 labels) ---
|
||||
# 5 labels should run in parallel, completing in ~0.1s (much faster than a sequential 0.5s)
|
||||
# Reset trackers
|
||||
active_requests = 0
|
||||
max_concurrent_requests = 0
|
||||
|
||||
start_time = time.time()
|
||||
labels_5 = [f"label_{i}" for i in range(5)]
|
||||
templates = ["Find {labels_str}"]
|
||||
|
||||
result_5 = await _run_object_detection(
|
||||
ctx=mock_ctx,
|
||||
image_path=str(dummy_image_path),
|
||||
queries=labels_5,
|
||||
templates=templates,
|
||||
)
|
||||
elapsed_5 = time.time() - start_time
|
||||
|
||||
# Concurrency assertions
|
||||
assert max_concurrent_requests == 5, (
|
||||
f"Expected 5 concurrent requests, got {max_concurrent_requests}"
|
||||
)
|
||||
assert elapsed_5 < 0.3, f"Expected execution to take < 0.3s, but took {elapsed_5:.3f}s"
|
||||
|
||||
# Verify result structure and swapped coordinates [y, x] -> [x, y]
|
||||
parsed_result_5 = result_5
|
||||
assert len(parsed_result_5["detected"]) == 5
|
||||
for item in parsed_result_5["detected"]:
|
||||
assert item["point"] == [500, 500]
|
||||
|
||||
# --- TEST 2: Throttling Assertion (10 labels) ---
|
||||
# 10 labels should be throttled by the semaphore limit of 6.
|
||||
# It should run in two batches, completing in ~0.2s (much faster than a sequential 1.0s).
|
||||
# Reset trackers
|
||||
active_requests = 0
|
||||
max_concurrent_requests = 0
|
||||
|
||||
start_time = time.time()
|
||||
labels_10 = [f"label_{i}" for i in range(10)]
|
||||
|
||||
result_10 = await _run_object_detection(
|
||||
ctx=mock_ctx,
|
||||
image_path=str(dummy_image_path),
|
||||
queries=labels_10,
|
||||
templates=templates,
|
||||
)
|
||||
elapsed_10 = time.time() - start_time
|
||||
|
||||
# Throttling assertions
|
||||
assert max_concurrent_requests == 6, (
|
||||
f"Expected max concurrent requests to be capped at 6, got {max_concurrent_requests}"
|
||||
)
|
||||
assert elapsed_10 < 0.5, (
|
||||
f"Expected execution to take < 0.5s (two batches of 0.1s), but took {elapsed_10:.3f}s"
|
||||
)
|
||||
|
||||
parsed_result_10 = result_10
|
||||
assert len(parsed_result_10["detected"]) == 10
|
||||
for item in parsed_result_10["detected"]:
|
||||
assert item["point"] == [500, 500]
|
||||
@@ -0,0 +1,126 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Ensure we can import from project
|
||||
sys.path.append(os.path.dirname(__file__))
|
||||
|
||||
from artemis.sdk import Agent
|
||||
from artemis.sdk.builders import Builders
|
||||
from artemis.sdk.types import AgentProfile
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.agents.validator.failure_analyzer import FailureAnalyzer, ValidationErrorCategory
|
||||
from artemis.controllers.unified_controller import UnifiedMobileController
|
||||
from artemis.utils.notes import get_note_file_path
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger("test_fa_standalone")
|
||||
|
||||
|
||||
async def main():
|
||||
load_dotenv()
|
||||
|
||||
# 1. Initialize Agent to set up device and connection pools
|
||||
logger.info("Initializing Agent and connecting to device...")
|
||||
profile = AgentProfile(name="default", from_file="llm-config.override.jsonc")
|
||||
config = Builders.AgentConfig.with_default_profile(profile).build()
|
||||
|
||||
agent = Agent(config=config)
|
||||
await agent.init()
|
||||
|
||||
# Create the ArtemisContext manually using agent's initialized clients
|
||||
context = ArtemisContext(
|
||||
device=agent._device_context,
|
||||
adb_client=agent._adb_client,
|
||||
ui_adb_client=agent._ui_adb_client,
|
||||
llm_config=profile.llm_config,
|
||||
agent_config=config,
|
||||
)
|
||||
|
||||
# Set up trace session and data engine
|
||||
from artemis.data_engine.engine import DataEngine
|
||||
|
||||
data_engine = DataEngine(ctx=context)
|
||||
context.data_engine = data_engine
|
||||
data_engine.start_session(goal="Play a video in full screen on YouTube in Chrome")
|
||||
|
||||
# Write the task plan note
|
||||
task_plan_content = (
|
||||
"- [ ] Open Chrome and navigate to the YouTube website (youtube.com)\n"
|
||||
"- [ ] Select and play any video\n"
|
||||
"- [ ] Make the playing video full screen"
|
||||
)
|
||||
task_plan_path = get_note_file_path(data_engine.base_dir, "task_plan")
|
||||
task_plan_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
task_plan_path.write_text(task_plan_content, encoding="utf-8")
|
||||
|
||||
# 2. Get initial screenshot
|
||||
controller = UnifiedMobileController(context)
|
||||
screen_data = await controller.controller.get_screen_data()
|
||||
screenshot_b64 = screen_data.base64
|
||||
|
||||
# Set up State
|
||||
state = State(
|
||||
messages=[],
|
||||
initial_goal="Play a video in full screen on YouTube in Chrome",
|
||||
latest_ui_hierarchy=screen_data.elements,
|
||||
latest_screenshot=str(
|
||||
data_engine.get_image_path(
|
||||
data_engine.get_or_create_image(base64.b64decode(screenshot_b64))
|
||||
)
|
||||
),
|
||||
focused_app_info="",
|
||||
device_date="",
|
||||
structured_decisions="",
|
||||
complete_subgoals_by_ids=[],
|
||||
validator_messages=[],
|
||||
)
|
||||
|
||||
# 3. Instantiate and run Failure Analyzer directly
|
||||
logger.info("Starting Failure Analyzer directly...")
|
||||
analyzer = FailureAnalyzer(context)
|
||||
analyzer.max_iterations = 45
|
||||
|
||||
# We pass a dummy failed action representing the fullscreen button tap
|
||||
dummy_failed_action = {
|
||||
"action": "tap",
|
||||
"coordinates": [968, 883],
|
||||
"target_text": "fullscreen button",
|
||||
}
|
||||
|
||||
try:
|
||||
result = await analyzer.analyze(
|
||||
state=state,
|
||||
failed_action=dummy_failed_action,
|
||||
error_msg="Simulated launch failure to trigger standalone FA run",
|
||||
pre_screenshot=screenshot_b64,
|
||||
post_screenshot=screenshot_b64,
|
||||
error_category=ValidationErrorCategory.TARGET_DISAPPEARED,
|
||||
)
|
||||
logger.info(f"Failure Analyzer direct execution finished! Result: {result}")
|
||||
except Exception:
|
||||
logger.exception("Error during Failure Analyzer execution:")
|
||||
finally:
|
||||
await agent.clean()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,58 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Ensure we can import from project
|
||||
sys.path.append(os.path.dirname(__file__))
|
||||
|
||||
from artemis.sdk import Agent
|
||||
from artemis.sdk.builders import Builders
|
||||
from artemis.sdk.types import AgentProfile
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger("test_flash_standalone")
|
||||
|
||||
|
||||
async def main():
|
||||
load_dotenv()
|
||||
|
||||
# 1. Initialize Agent to set up device and connection pools
|
||||
logger.info("Initializing Agent and connecting to device...")
|
||||
profile = AgentProfile(name="default", from_file="llm-config.override.jsonc")
|
||||
config = Builders.AgentConfig.with_default_profile(profile).build()
|
||||
|
||||
agent = Agent(config=config)
|
||||
await agent.init()
|
||||
|
||||
# 2. Run task using 'flash' profile
|
||||
logger.info("Starting Flash runner task execution...")
|
||||
try:
|
||||
result = await agent.run_task(
|
||||
goal="Open USA Today in Chrome browser and view the USA news",
|
||||
profile="flash",
|
||||
)
|
||||
logger.info(f"Flash runner task execution finished! Result: {result}")
|
||||
except Exception:
|
||||
logger.exception("Error during Flash task execution:")
|
||||
finally:
|
||||
await agent.clean()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,57 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Ensure we can import from project
|
||||
sys.path.append(os.path.dirname(__file__))
|
||||
|
||||
from artemis.sdk import Agent
|
||||
from artemis.sdk.builders import Builders
|
||||
from artemis.sdk.types import AgentProfile
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger("test_flash_standalone")
|
||||
|
||||
|
||||
async def main():
|
||||
load_dotenv()
|
||||
|
||||
# 1. Initialize Agent to set up device and connection pools
|
||||
logger.info("Initializing Agent and connecting to device...")
|
||||
profile = AgentProfile(name="default", from_file="llm-config.override.jsonc")
|
||||
config = Builders.AgentConfig.with_default_profile(profile).build()
|
||||
|
||||
agent = Agent(config=config)
|
||||
await agent.init()
|
||||
|
||||
# 2. Run task using 'flash' profile
|
||||
logger.info("Starting Flash runner task execution...")
|
||||
try:
|
||||
result = await agent.run_task(
|
||||
goal="Play a video in full screen on YouTube in Chrome", profile="flash"
|
||||
)
|
||||
logger.info(f"Flash runner task execution finished! Result: {result}")
|
||||
except Exception:
|
||||
logger.exception("Error during Flash task execution:")
|
||||
finally:
|
||||
await agent.clean()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,21 @@
|
||||
import time
|
||||
import uiautomator2 as u2
|
||||
|
||||
|
||||
def test():
|
||||
device = u2.connect("emulator-5554")
|
||||
print("Testing send_keys with immediate IME switch off")
|
||||
device.set_fastinput_ime(True)
|
||||
device.send_keys("Test 1")
|
||||
device.set_fastinput_ime(False)
|
||||
|
||||
time.sleep(2)
|
||||
print("Testing send_keys with delay before IME switch off")
|
||||
device.set_fastinput_ime(True)
|
||||
device.send_keys("Test 2")
|
||||
time.sleep(1.0)
|
||||
device.set_fastinput_ime(False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test()
|
||||
@@ -0,0 +1,102 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
# Add root to sys.path
|
||||
root_dir = Path(__file__).parent.parent
|
||||
if str(root_dir) not in sys.path:
|
||||
sys.path.insert(0, str(root_dir))
|
||||
|
||||
from adbutils import AdbClient
|
||||
from artemis.clients.ui_automator_client import UIAutomatorClient
|
||||
from artemis.context import DeviceContext, DevicePlatform, ArtemisContext
|
||||
from artemis.controllers.unified_controller import UnifiedMobileController
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_actions():
|
||||
print("=========================================")
|
||||
print(" Physical Action Direct Validation Testbench")
|
||||
print("=========================================")
|
||||
adb = AdbClient(host="localhost", port=5037)
|
||||
devices = adb.device_list()
|
||||
if not devices:
|
||||
print("❌ Error: No active Android emulator or physical device detected.")
|
||||
return
|
||||
device_id = devices[0].serial
|
||||
print(f"✅ Target device locked: {device_id}")
|
||||
|
||||
ui_client = UIAutomatorClient(device_id=device_id)
|
||||
print("🔄 Connecting to UIAutomator2 and syncing screen properties...")
|
||||
try:
|
||||
ui_data = ui_client.get_screen_data()
|
||||
w, h = ui_data.width, ui_data.height
|
||||
print(f"✅ Screen sync ready, physical resolution: {w}x{h}")
|
||||
except Exception as e:
|
||||
print(f"❌ Screen properties sync failed: {e}")
|
||||
return
|
||||
|
||||
ctx = ArtemisContext(
|
||||
trace_id="physical-test-bench",
|
||||
device=DeviceContext(
|
||||
host_platform="LINUX",
|
||||
mobile_platform=DevicePlatform.ANDROID,
|
||||
device_id=device_id,
|
||||
device_width=w,
|
||||
device_height=h,
|
||||
),
|
||||
adb_client=adb,
|
||||
ui_adb_client=ui_client,
|
||||
)
|
||||
|
||||
ctrl = UnifiedMobileController(ctx)
|
||||
|
||||
print("\n[Task 1] Extract Live UI Hierarchy...")
|
||||
try:
|
||||
elems = await ctrl.get_ui_elements()
|
||||
print(f"✅ Parsed UI element count: {len(elems)}")
|
||||
except Exception as e:
|
||||
print(f"❌ Failed to extract UI hierarchy: {e}")
|
||||
|
||||
print("\n[Task 2] Capture Live Screenshot...")
|
||||
try:
|
||||
shot = await ctrl.take_screenshot()
|
||||
print(f"✅ Screenshot captured, Base64 payload length: {len(shot)}")
|
||||
except Exception as e:
|
||||
print(f"❌ Failed to capture screenshot: {e}")
|
||||
|
||||
print("\n[Task 3] Simulate Drag/Swipe Down...")
|
||||
try:
|
||||
# Swipe up from bottom to top
|
||||
err = await ctrl.swipe_coords(w // 2, int(h * 0.8), w // 2, int(h * 0.2), duration=400)
|
||||
if err:
|
||||
print(f"❌ Swipe rejected by controller: {err}")
|
||||
else:
|
||||
print("✅ Swipe command injected successfully!")
|
||||
except Exception as e:
|
||||
print(f"❌ Swipe execution exception: {e}")
|
||||
|
||||
print("\n🧹 Safely releasing connection and driver handles...")
|
||||
await ctrl.cleanup()
|
||||
print("🎉 Direct physical validation tests completed successfully!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_actions())
|
||||
@@ -0,0 +1,45 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Integration tests for ReactiveRunner and Pipeline."""
|
||||
|
||||
import pytest
|
||||
from artemis.core.context import ExecutionContext
|
||||
from artemis.core.state import ExecutionStatus
|
||||
from artemis.drivers.mock.mock_driver import MockDeviceDriver
|
||||
from artemis.engine.pipeline import Pipeline
|
||||
from artemis.engine.reactive_runner import ReactiveRunner
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reactive_runner_execution():
|
||||
"""Verify ReactiveRunner executes workflow cycles on mock driver."""
|
||||
ctx = ExecutionContext(task_goal="Turn on Bluetooth in Settings", device_id="test-emulator")
|
||||
driver = MockDeviceDriver(device_id="test-emulator")
|
||||
|
||||
runner = ReactiveRunner(ctx=ctx, driver=driver)
|
||||
state = await runner.run(max_turns=5)
|
||||
|
||||
assert state.status == ExecutionStatus.SUCCESS
|
||||
assert state.current_turn >= 1
|
||||
assert len(state.steps) >= 1
|
||||
assert state.steps[0].action_name == "wait_for_delay"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pipeline_dispatch():
|
||||
"""Verify Pipeline executes with default mock driver."""
|
||||
state = await Pipeline.execute(goal="Open YouTube and play video", profile="flash")
|
||||
assert state.status == ExecutionStatus.SUCCESS
|
||||
assert state.task_goal == "Open YouTube and play video"
|
||||
Reference in new issue
Block a user