chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

feat: improve task execution, verification and packaging

Browse Source 
Add execution incidents, action bursts and shared Flash history.

Expose verification and Explorer settings with streamed check results.

Support graceful cancellation, recording recovery and bundled UI resources.

Add CI quality checks and clean up comments and documentation.
This commit is contained in:
somew1nd committed 2026-09-02 21:45:51 -07:00
1 parent 5ccf06d677
commit 67248c887e
289 files changed
+15883 -6919
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/conftest.py
+20 -2
View File
@@ -12,13 +12,31 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Global pytest fixtures for ARTEMIS test suite."""
"""Repository-wide pytest fixtures and classification helpers.
The default test paths contain only deterministic tests. Tests under the
integration and end-to-end trees remain directly runnable, and receive stable
markers here so callers can select them without relying on filename patterns.
"""
from pathlib import Path
import pytest
from artemis.drivers.mock.mock_driver import MockDeviceDriver
@pytest.fixture
def mock_driver():
"""Provides an isolated MockDeviceDriver instance."""
"""Provide an isolated mock mobile driver."""
return MockDeviceDriver(device_id="fixture-mock-device", width=1080, height=2400)
def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:
"""Attach test-layer markers according to the owning test directory."""
for item in items:
parts = Path(str(item.path)).parts
if "integration" in parts:
item.add_marker(pytest.mark.integration)
if "e2e" in parts:
item.add_marker(pytest.mark.e2e)
tests/e2e/test_failure_analyzer_e2e_youtube.py
-124
View File
@@ -1,124 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import logging
import os
import sys
from unittest.mock import patch
from dotenv import load_dotenv
# Ensure we can import from project
sys.path.append(os.path.dirname(__file__))
from artemis.sdk import Agent
from artemis.sdk.builders import Builders
from artemis.sdk.types import AgentProfile
from artemis.agents.validator.validator import ValidatorNode, ValidationErrorCategory
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("test_fa_e2e")
# We want to fail the first validation to trigger the Failure Analyzer
original_validate = ValidatorNode._validate_action_precondition
original_validate_pixel = ValidatorNode._validate_action_precondition_pixel
has_failed_once = False
async def mock_validate_action_precondition(self, session, action_item, state=None):
global has_failed_once
if not has_failed_once:
has_failed_once = True
logger.warning(">>> MOCK VALIDATOR: Simulating pre-execution XML validation failure!")
return (
False,
ValidationErrorCategory.TARGET_DISAPPEARED,
"Simulated pre-execution XML validation failure",
)
return await original_validate(self, session, action_item, state=state)
async def mock_validate_action_precondition_pixel(
self,
session,
action_item,
pre_screenshot_b64,
original_coords=None,
state=None,
):
global has_failed_once
if not has_failed_once:
has_failed_once = True
logger.warning(">>> MOCK VALIDATOR: Simulating pre-execution Pixel validation failure!")
return (
False,
ValidationErrorCategory.TARGET_DISAPPEARED,
"Simulated pre-execution Pixel validation failure",
)
return await original_validate_pixel(
self,
session,
action_item,
pre_screenshot_b64,
original_coords,
state=state,
)
async def main():
load_dotenv()
# Check if emulator/device is connected
import subprocess
res = subprocess.run(["adb", "devices"], capture_output=True, text=True)
print("ADB Devices:")
print(res.stdout)
# 1. Initialize agent
logger.info("Initializing Agent...")
profile = AgentProfile(name="default", from_file="llm-config.override.jsonc")
config = Builders.AgentConfig.with_default_profile(profile).build()
agent = Agent(config=config)
await agent.init()
# 2. Run the task with patches
goal = "Play a video in full screen on YouTube in Chrome"
logger.info(f"Running task with goal: {goal}")
with (
patch.object(
ValidatorNode,
"_validate_action_precondition",
mock_validate_action_precondition,
),
patch.object(
ValidatorNode,
"_validate_action_precondition_pixel",
mock_validate_action_precondition_pixel,
),
):
try:
result = await agent.run_task(goal=goal, name="test_fa_youtube_fullscreen")
logger.info(f"Task finished! Result: {result}")
except Exception:
logger.exception("Error during execution:")
finally:
await agent.clean()
if __name__ == "__main__":
asyncio.run(main())
tests/e2e/test_multi_port_end_to_end.py
+15 -5
View File
@@ -95,7 +95,9 @@ def test_02_daemon_device_discovery_sees_phone_and_emulator():
# Must detect both devices
assert any(s.startswith("emulator") for s in serials), f"Emulator not found in {serials}"
assert any(not s.startswith("emulator") for s in serials), f"Physical phone not found in {serials}"
assert any(not s.startswith("emulator") for s in serials), (
f"Physical phone not found in {serials}"
)
# Check is_emulator flag classification
for d in devices:
@@ -182,7 +184,9 @@ def test_05_sdk_client_targeting_and_concurrency():
assert acq1 is True, "Failed to acquire emulator lock"
try:
acq2 = lock_phone.acquire(timeout=5.0)
assert acq2 is True, "Failed to acquire phone lock concurrently (multi-device parallel mode)"
assert acq2 is True, (
"Failed to acquire phone lock concurrently (multi-device parallel mode)"
)
lock_phone.release()
finally:
lock_emu.release()
@@ -191,18 +195,24 @@ def test_05_sdk_client_targeting_and_concurrency():
def test_06_cli_daemon_and_standalone_options():
"""Verify CLI commands properly expose and route through Daemon or standalone."""
# artemis status
res = subprocess.run([sys.executable, "-m", "artemis.main", "status"], capture_output=True, text=True)
res = subprocess.run(
[sys.executable, "-m", "artemis.main", "status"], capture_output=True, text=True
)
assert res.returncode == 0
assert "online" in res.stdout.lower() or "running" in res.stdout.lower()
# artemis run --help exposes --standalone and --device
res_run = subprocess.run([sys.executable, "-m", "artemis.main", "run", "--help"], capture_output=True, text=True)
res_run = subprocess.run(
[sys.executable, "-m", "artemis.main", "run", "--help"], capture_output=True, text=True
)
assert res_run.returncode == 0
assert "--standalone" in res_run.stdout
assert "--device" in res_run.stdout
# artemis batch --help exposes --standalone
res_batch = subprocess.run([sys.executable, "-m", "artemis.main", "batch", "--help"], capture_output=True, text=True)
res_batch = subprocess.run(
[sys.executable, "-m", "artemis.main", "batch", "--help"], capture_output=True, text=True
)
assert res_batch.returncode == 0
assert "--standalone" in res_batch.stdout
tests/fixtures/action_surfaces/operator_shells.json
Vendored
+2 -2
View File
@@ -187,7 +187,7 @@
}
],
"default": null,
"description": "Optional swipe/drag duration in milliseconds (default 800). For drag-and-drop, list reordering, or sliding/adjusting sliders (e.g., volume, brightness, SeekBars), set duration >= 1000 (e.g. 1500). If omitted for directional swipe, duration is computed automatically."
"description": "Optional swipe/drag duration in milliseconds (default 800; computed automatically for directional swipes)."
}
},
"type": "object"
@@ -198,7 +198,7 @@
"type": "function",
"function": {
"name": "press_key",
"description": "[ACTION] Press a physical or virtual system button (e.g.\n\nENTER, BACK, HOME, APP_SWITCH).",
"description": "[ACTION] Press a physical or virtual system button (e.g. ENTER, BACK, HOME, APP_SWITCH).",
"parameters": {
"properties": {
"key": {
tests/integration/agents/explorer/test_explorer_all_tools/test_explorer_all_tools.py
+19 -31
View File
@@ -214,26 +214,22 @@ async def test_explorer_all_tools_sequential_mocked():
]
mock_storage.get_image.return_value = mock_record
# Mock low-level layout searching (search_ui_func)
mock_search_ui = MagicMock(
return_value={
"matches": [
{
"matched_text": "Settings Button",
"bounds": [400, 400, 600, 600],
}
]
# The UI tree of the record feeds the in-memory ScreenIndex that backs the
# text search and the coordinate audit (no Data Engine query per tool).
mock_record.ui_tree = [
{
"text": "Settings Button",
"bounds": "[400,400][600,600]",
"class": "android.widget.Button",
"clickable": "true",
}
)
]
# Mock downstream object detection pipeline (_run_object_detection)
mock_detect = AsyncMock(
return_value={"detected": [{"label": "gear icon", "point": [200, 300]}]}
)
# Mock spatial coordinate lookup (search_by_coordinates_func)
mock_lookup = MagicMock(return_value="Matching Elements: Container/DashboardFrame")
logger.info("Starting Explorer.run execution loop with sequential tool mocks...")
with (
@@ -245,12 +241,9 @@ async def test_explorer_all_tools_sequential_mocked():
"artemis.agents.explorer.explorer.StorageManager",
return_value=mock_storage,
),
patch("artemis.agents.explorer.explorer.search_ui_func", mock_search_ui),
# get_ocr_list is only exposed when an OCR backend is configured.
patch("artemis.agents.explorer.explorer.is_ocr_configured", return_value=True),
patch("artemis.agents.explorer.explorer._run_object_detection", mock_detect),
patch(
"artemis.agents.explorer.explorer.search_by_coordinates_func",
mock_lookup,
),
):
explorer = Explorer(mock_ctx)
result = await explorer.run(
@@ -259,6 +252,7 @@ async def test_explorer_all_tools_sequential_mocked():
screenshot_path=str(input_screenshot),
state=mock_state,
version="ultra",
enable_caching=False,
)
# Verify ReAct loop went through all 5 turns
@@ -288,14 +282,14 @@ async def test_explorer_all_tools_sequential_mocked():
# 1. Verify get_ocr_list
ocr_trace = explorer.trace_history[0]["tool_calls"][0]
assert ocr_trace["name"] == "get_ocr_list"
assert "[T1]" in ocr_trace["response"]["text"]
assert "[O" in ocr_trace["response"]["text"]
assert "[462,208]" in ocr_trace["response"]["text"]
assert ocr_trace["response"]["image_path"] is not None
# 2. Verify ask_perception_tool
search_trace = explorer.trace_history[1]["tool_calls"][0]
assert search_trace["name"] == "ask_perception_tool"
assert "XML/OCR Text Search Results" in search_trace["response"]["text"]
assert "Text Search Results are" in search_trace["response"]["text"]
assert len(search_trace["response"]["image_paths"]) > 0
# 3. Verify detect_objects
@@ -308,17 +302,11 @@ async def test_explorer_all_tools_sequential_mocked():
coords_trace = explorer.trace_history[3]["tool_calls"][0]
assert coords_trace["name"] == "inspect_region"
assert coords_trace["response"]["image_path"] is not None
# Compute expected screenshot hash
import hashlib
sha256_hash = hashlib.sha256()
with open(input_screenshot, "rb") as f:
for byte_block in iter(lambda: f.read(4096), b""):
sha256_hash.update(byte_block)
expected_hash = sha256_hash.hexdigest()
# Should map normalized [500, 500] to pixel [540, 1200]
mock_lookup.assert_called_once_with(expected_hash, 540, 1200)
# The coordinate audit at normalized [500, 500] (pixel [540, 1200]) is
# answered from the in-memory index: the UI-tree button covers that point.
perception_trace = explorer.trace_history[1]["tool_calls"][0]
assert perception_trace["name"] == "ask_perception_tool"
assert "Settings Button" in perception_trace["response"]["text"]
logger.info(
"All assertions passed! The entire suite of explorer tools is fully functional."
tests/integration/agents/explorer/test_explorer_all_tools/test_explorer_ask_image_processor.py
+1 -1
View File
@@ -194,7 +194,7 @@ async def test_explorer_ask_image_processor_tool():
return_value=mock_client,
),
patch(
"artemis.agents.image_processor.image_processor.ImageProcessor",
"artemis.agents.explorer.perception_tools.ImageProcessor",
return_value=mock_image_processor_instance,
),
):
tests/integration/test_artemis_login.py
+8 -1
View File
@@ -13,12 +13,19 @@
# limitations under the License.
import os
import pty
import select
import sys
import time
import pytest
pytestmark = [pytest.mark.integration, pytest.mark.manual]
if sys.platform == "win32":
pytest.skip("PTY-based SSH login test requires a POSIX host", allow_module_level=True)
import pty
@pytest.mark.skip(reason="Manual integration test requiring remote host")
def test_artemis_ssh_login():
tests/integration/test_cloud_gateway.py
+6
View File
@@ -22,6 +22,12 @@ workspace path resolution, RESTful /api/v1/* endpoints, and SSE event streaming.
import pytest
from fastapi.testclient import TestClient
pytestmark = [pytest.mark.integration, pytest.mark.cloud]
pytest.importorskip(
"cloud_service",
reason="optional cloud_service package is not installed",
)
from cloud_service.gateway.auth import TokenAuthProvider
from cloud_service.gateway.manager import ConnectionManager
tests/integration/test_failure_analyzer_standalone_youtube.py
-121
View File
@@ -1,121 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import base64
import logging
import os
import sys
from dotenv import load_dotenv
# Ensure we can import from project
sys.path.append(os.path.dirname(__file__))
from artemis.sdk import Agent
from artemis.sdk.builders import Builders
from artemis.sdk.types import AgentProfile
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.agents.validator.failure_analyzer import FailureAnalyzer, ValidationErrorCategory
from artemis.controllers.unified_controller import UnifiedMobileController
from artemis.utils.notes import get_note_file_path
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("test_fa_standalone")
async def main():
load_dotenv()
# 1. Initialize Agent to set up device and connection pools
logger.info("Initializing Agent and connecting to device...")
profile = AgentProfile(name="default", from_file="llm-config.override.jsonc")
config = Builders.AgentConfig.with_default_profile(profile).build()
agent = Agent(config=config)
await agent.init()
# Create the ArtemisContext manually using agent's initialized clients
context = ArtemisContext(
device=agent._device_context,
adb_client=agent._adb_client,
ui_adb_client=agent._ui_adb_client,
llm_config=profile.llm_config,
agent_config=config,
)
# Set up trace session and data engine
from artemis.data_engine.engine import DataEngine
data_engine = DataEngine(ctx=context)
context.data_engine = data_engine
data_engine.start_session(goal="Play a video in full screen on YouTube in Chrome")
# Write the task plan note
task_plan_content = (
"- [ ] Open Chrome and navigate to the YouTube website (youtube.com)\n"
"- [ ] Select and play any video\n"
"- [ ] Make the playing video full screen"
)
task_plan_path = get_note_file_path(data_engine.base_dir, "task_plan")
task_plan_path.parent.mkdir(parents=True, exist_ok=True)
task_plan_path.write_text(task_plan_content, encoding="utf-8")
# 2. Get initial screenshot
controller = UnifiedMobileController(context)
screen_data = await controller.controller.get_screen_data()
screenshot_b64 = screen_data.base64
# Set up State
state = State(
initial_goal="Play a video in full screen on YouTube in Chrome",
latest_ui_hierarchy=screen_data.elements,
latest_screenshot=str(
data_engine.get_image_path(
data_engine.get_or_create_image(base64.b64decode(screenshot_b64))
)
),
structured_decisions="",
)
# 3. Instantiate and run Failure Analyzer directly
logger.info("Starting Failure Analyzer directly...")
analyzer = FailureAnalyzer(context)
analyzer.max_iterations = 45
# We pass a dummy failed action representing the fullscreen button tap
dummy_failed_action = {
"action": "tap",
"coordinates": [968, 883],
"target_text": "fullscreen button",
}
try:
result = await analyzer.analyze(
state=state,
failed_action=dummy_failed_action,
error_msg="Simulated launch failure to trigger standalone FA run",
pre_screenshot=screenshot_b64,
post_screenshot=screenshot_b64,
error_category=ValidationErrorCategory.TARGET_DISAPPEARED,
)
logger.info(f"Failure Analyzer direct execution finished! Result: {result}")
except Exception:
logger.exception("Error during Failure Analyzer execution:")
finally:
await agent.clean()
if __name__ == "__main__":
asyncio.run(main())
tests/integration/test_physical_actions.py
+3 -2
View File
@@ -29,6 +29,8 @@ from artemis.controllers.unified_controller import UnifiedMobileController
import pytest
pytestmark = [pytest.mark.integration, pytest.mark.android]
@pytest.mark.asyncio
async def test_actions():
@@ -38,8 +40,7 @@ async def test_actions():
adb = AdbClient(host="localhost", port=5037)
devices = adb.device_list()
if not devices:
print("❌ Error: No active Android emulator or physical device detected.")
return
pytest.skip("No active Android emulator or physical device detected")
device_id = devices[0].serial
print(f"✅ Target device locked: {device_id}")
tests/integration/test_remote_adb.py
+8
View File
@@ -18,6 +18,14 @@ Unit tests for RemoteAdbDevice, RemoteAdbClient, and platform specific commands
from unittest.mock import MagicMock
import pytest
pytestmark = [pytest.mark.integration, pytest.mark.cloud]
pytest.importorskip(
"cloud_service",
reason="optional cloud_service package is not installed",
)
from cloud_service.virtualization.remote_adb import (
RemoteAdbDevice,
RemoteAdbClient,
tests/tools/explorer/test_explorer_tools.py
+3
View File
@@ -38,6 +38,7 @@ def explorer_instance(artemis_context, mock_state):
@pytest.mark.asyncio
@pytest.mark.integration
async def test_exec_detect_objects(explorer_instance):
result = await explorer_instance.exec_detect_objects(
["button", "text"], target_image_id="img_0"
@@ -48,6 +49,7 @@ async def test_exec_detect_objects(explorer_instance):
@pytest.mark.asyncio
@pytest.mark.integration
async def test_exec_ask_perception_tool(explorer_instance):
result = await explorer_instance.exec_ask_perception_tool("login", 500, 500, ["button"])
assert isinstance(result, dict)
@@ -74,6 +76,7 @@ async def test_exec_inspect_region(explorer_instance):
@pytest.mark.asyncio
@pytest.mark.integration
async def test_exec_ask_image_processor(explorer_instance):
result = await explorer_instance.exec_ask_image_processor(
"Invert colors", target_image_id="img_0"
tests/tools/history_analyzer/test_history_analyzer_tools.py
+1
View File
@@ -80,6 +80,7 @@ async def test_get_step_details_tool_with_fixture(artemis_context):
@pytest.mark.asyncio
@pytest.mark.integration
async def test_history_analyzer_blackbox_run(artemis_context):
"""End-to-end blackbox test of HistoryAnalyzer.run(query) using real steps from inputs/data_engine.db and real LLM calls."""
if not artemis_context.data_engine:
tests/tools/image_processor/test_image_processor_tools.py
+1
View File
@@ -17,6 +17,7 @@ import pytest
@pytest.mark.asyncio
@pytest.mark.integration
async def test_ask_image_processor_tool(artemis_context, mock_state):
"""Tests the ask_image_processor tool as a blackbox.
tests/tools/log_analyzer/test_log_analyzer_tools.py
+2
View File
@@ -17,6 +17,7 @@ import pytest
@pytest.mark.asyncio
@pytest.mark.integration
async def test_spawn_log_reader(artemis_context, mock_state):
"""Test the spawn_log_reader tool exposed by LogAnalyzerNode."""
from artemis.sdk.utils import load_llm_config_override
@@ -42,6 +43,7 @@ async def test_spawn_log_reader(artemis_context, mock_state):
@pytest.mark.asyncio
@pytest.mark.integration
async def test_log_analyzer_run(artemis_context, mock_state):
"""Test LogAnalyzerNode.run end-to-end execution with grounding configuration."""
from artemis.config.llm import parse_llm_config
tests/tools/validator/test_validator_tools.py
+1 -1
View File
@@ -15,7 +15,7 @@
import base64
from pathlib import Path
from artemis.agents.validator.failure_analyzer import ValidationErrorCategory
from artemis.agents.validator.categories import ValidationErrorCategory
from artemis.agents.validator.validator import ValidatorNode
import pytest
tests/tools/video_analyzer/test_video_analyzer_tools.py
+1
View File
@@ -20,6 +20,7 @@ from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer
@pytest.mark.asyncio
@pytest.mark.integration
async def test_video_analyzer_tools(artemis_context, mock_state, inputs_dir, tmp_path):
artemis_context.device.mobile_platform = "android"
artemis_context.adb_client = artemis_context.ui_adb_client
tests/unit/admin_console/test_graceful_stop_and_recovery.py
+273
View File
@@ -0,0 +1,273 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Cancelling a task must let the worker finalize its recording, or recover it."""
import asyncio
from contextlib import ExitStack
from unittest.mock import MagicMock, patch
import pytest
from apps.admin_console.core.state import state
from apps.admin_console.services.task_queue_service import TaskQueueService, task_queue_service
from artemis.runtime import cancel_requests
from artemis.runtime.device_lock import DeviceLockOwner
TQS = "apps.admin_console.services.task_queue_service"
def _reset_state() -> None:
state.clear_queue()
state.queue_items.clear()
state.active_connections.clear()
state.cancelled_session_ids.clear()
state.manually_stopped_run_ids.clear()
state.current_process = None
state.active_session_id = None
state.current_goal = None
state.current_profile = None
@pytest.fixture(autouse=True)
def isolated(tmp_path, monkeypatch):
marker_dir = tmp_path / "cancel-requests"
lock_dir = tmp_path / "device-locks"
lock_dir.mkdir()
monkeypatch.setattr(cancel_requests, "get_temp_dir", lambda _subfolder=None: marker_dir)
monkeypatch.setattr(
"artemis.runtime.device_lock.get_temp_dir", lambda _subfolder=None: lock_dir
)
monkeypatch.setenv("ARTEMIS_CANCEL_GRACE_SECONDS", "5")
for task in list(TaskQueueService._forced_stop_tasks):
task.cancel()
TaskQueueService._forced_stop_tasks.clear()
_reset_state()
yield marker_dir
for task in list(TaskQueueService._forced_stop_tasks):
task.cancel()
TaskQueueService._forced_stop_tasks.clear()
_reset_state()
def _owner(pid: int = 24680, session_id: str = "sess-1") -> DeviceLockOwner:
return DeviceLockOwner(
pid=pid,
process_created_at=1234.5,
token="owner-token",
device_id="emulator-5554",
description="frontend task: demo",
acquired_at="2026-08-24T00:00:00+00:00",
session_id=session_id,
ingress="frontend",
)
def _stop_patches(stack: ExitStack, owner: DeviceLockOwner, alive: bool = True):
stack.enter_context(patch.object(TaskQueueService, "ensure_worker_running"))
stack.enter_context(
patch(f"{TQS}.DeviceExecutionLock.get_active_owners", return_value={"dev": owner})
)
stack.enter_context(patch(f"{TQS}.DeviceExecutionLock.is_active_owner", return_value=True))
stack.enter_context(patch(f"{TQS}.DeviceExecutionLock.cleanup_stale_locks"))
stack.enter_context(patch(f"{TQS}.session_repo.update_session_status"))
stack.enter_context(patch("artemis.runtime.trace_store.update_trace_status"))
pid_alive = stack.enter_context(patch(f"{TQS}.pid_is_alive", return_value=alive))
terminate = stack.enter_context(patch(f"{TQS}.process_supervisor.terminate_tree_verified"))
return pid_alive, terminate
@pytest.mark.asyncio
async def test_stop_requests_graceful_cancel_and_skips_kill_when_worker_exits():
owner = _owner()
with ExitStack() as stack:
pid_alive, terminate = _stop_patches(stack, owner)
assert task_queue_service.stop_tasks(session_id="sess-1") is True
# The worker is asked to cancel itself; nothing is killed yet.
terminate.assert_not_called()
assert cancel_requests.is_cancel_requested(session_id="sess-1")
assert cancel_requests.is_cancel_requested(pid=24680, process_created_at=1234.5)
assert len(TaskQueueService._forced_stop_tasks) == 1
enforcer = next(iter(TaskQueueService._forced_stop_tasks))
# The worker finalizes and exits on its own within the grace period.
pid_alive.return_value = False
await asyncio.wait_for(enforcer, timeout=5)
terminate.assert_not_called()
assert not cancel_requests.is_cancel_requested(session_id="sess-1")
@pytest.mark.asyncio
async def test_stop_kills_worker_that_ignores_the_cancel_request(monkeypatch):
monkeypatch.setenv("ARTEMIS_CANCEL_GRACE_SECONDS", "0.6")
owner = _owner()
with ExitStack() as stack:
_pid_alive, terminate = _stop_patches(stack, owner, alive=True)
terminate.return_value = True
assert task_queue_service.stop_tasks(session_id="sess-1") is True
terminate.assert_not_called()
enforcer = next(iter(TaskQueueService._forced_stop_tasks))
await asyncio.wait_for(enforcer, timeout=5)
terminate.assert_called_once_with(24680, 1234.5)
assert not cancel_requests.is_cancel_requested(session_id="sess-1")
def test_zero_grace_keeps_the_immediate_kill(monkeypatch):
monkeypatch.setenv("ARTEMIS_CANCEL_GRACE_SECONDS", "0")
owner = _owner()
with ExitStack() as stack:
_pid_alive, terminate = _stop_patches(stack, owner)
terminate.return_value = True
assert task_queue_service.stop_tasks(session_id="sess-1") is True
terminate.assert_called_once_with(24680, 1234.5)
assert not cancel_requests.is_cancel_requested(session_id="sess-1")
assert not TaskQueueService._forced_stop_tasks
def test_stop_of_dead_worker_writes_no_marker():
owner = _owner()
with ExitStack() as stack:
_pid_alive, terminate = _stop_patches(stack, owner, alive=False)
assert task_queue_service.stop_tasks(session_id="sess-1") is True
terminate.assert_not_called()
assert not cancel_requests.is_cancel_requested(session_id="sess-1")
assert not TaskQueueService._forced_stop_tasks
@pytest.mark.asyncio
async def test_recover_or_fail_recording_finalizes_orphaned_raw_file(tmp_path):
final = tmp_path / "web_x_FAIL_2026-09-02T16-31-32" / "recording.mp4"
final.parent.mkdir()
final.write_bytes(b"video")
raw_path = str(tmp_path / "web_x" / "recording.mkv")
repo = MagicMock()
repo.get_video_recording_for_session.return_value = {
"status": "recording",
"local_video_path": raw_path,
"start_time": 1788391895.4,
}
media = MagicMock()
media.recover_orphaned_recording.return_value = final
media.path_to_video_url.return_value = "/videos/traces/web_x_FAIL/recording.mp4"
events: list[tuple[str, dict]] = []
with (
patch(f"{TQS}.session_repo", repo),
patch(f"{TQS}.media_service", media),
patch.object(
TaskQueueService, "_broadcast_event", side_effect=lambda t, d: events.append((t, d))
),
):
await TaskQueueService._recover_or_fail_recording("sess-1")
media.recover_orphaned_recording.assert_called_once_with(raw_path, 1788391895.4)
repo.mark_recording_ready.assert_called_once_with("sess-1", str(final))
repo.mark_recording_failed_if_pending.assert_not_called()
assert events == [
(
"recording_ready",
{"session_id": "sess-1", "video_url": "/videos/traces/web_x_FAIL/recording.mp4"},
)
]
@pytest.mark.asyncio
async def test_recover_or_fail_recording_marks_failed_when_nothing_usable():
repo = MagicMock()
repo.get_video_recording_for_session.return_value = {
"status": "recording",
"local_video_path": "/nowhere/recording.mkv",
"start_time": 1.0,
}
repo.get_video_recordings_map.return_value = {}
repo.mark_recording_failed_if_pending.return_value = True
media = MagicMock()
media.recover_orphaned_recording.return_value = None
media.build_video_index.return_value = {}
media.resolve_video_url.return_value = None
events: list[tuple[str, dict]] = []
with (
patch(f"{TQS}.session_repo", repo),
patch(f"{TQS}.media_service", media),
patch.object(
TaskQueueService, "_broadcast_event", side_effect=lambda t, d: events.append((t, d))
),
):
await TaskQueueService._recover_or_fail_recording("sess-2")
repo.mark_recording_ready.assert_not_called()
repo.mark_recording_failed_if_pending.assert_called_once()
assert events[0][0] == "recording_failed"
def test_startup_sweep_publishes_orphaned_recordings(tmp_path):
final = tmp_path / "web_a_FAIL_2026-09-02T16-31-32" / "recording.mp4"
final.parent.mkdir()
final.write_bytes(b"video")
repo = MagicMock()
repo.get_unfinalized_video_recordings.return_value = [
{
"session_id": "sess-a",
"local_video_path": str(tmp_path / "web_a" / "recording.mkv"),
"start_time": 10.0,
"session_status": "cancelled",
},
{
"session_id": "sess-b",
"local_video_path": str(tmp_path / "web_b" / "recording.mkv"),
"start_time": None,
"session_start_time": 20.0,
"session_status": "failed",
},
]
repo.mark_recording_ready.return_value = True
media = MagicMock()
media.recover_orphaned_recording.side_effect = [final, None]
with patch(f"{TQS}.session_repo", repo), patch(f"{TQS}.media_service", media):
assert TaskQueueService.recover_orphaned_recordings_on_launch() == 1
assert media.recover_orphaned_recording.call_args_list[0].args == (
str(tmp_path / "web_a" / "recording.mkv"),
10.0,
)
assert media.recover_orphaned_recording.call_args_list[1].args == (
str(tmp_path / "web_b" / "recording.mkv"),
20.0,
)
repo.mark_recording_ready.assert_called_once_with("sess-a", str(final))
@pytest.mark.asyncio
async def test_recover_or_fail_recording_skips_rows_the_worker_finalized():
repo = MagicMock()
repo.get_video_recording_for_session.return_value = {"status": "ready"}
media = MagicMock()
with patch(f"{TQS}.session_repo", repo), patch(f"{TQS}.media_service", media):
await TaskQueueService._recover_or_fail_recording("sess-3")
media.recover_orphaned_recording.assert_not_called()
repo.mark_recording_failed_if_pending.assert_not_called()
tests/unit/admin_console/test_media_recording_lifecycle.py
+7 -1
View File
@@ -51,7 +51,13 @@ async def test_session_video_publishes_only_finalized_versioned_media(monkeypatc
"resolve_video_segments",
MagicMock(
return_value=[
{"url": "/videos/recording.mp4", "start": 0, "duration": 4, "width": 1080, "height": 1920}
{
"url": "/videos/recording.mp4",
"start": 0,
"duration": 4,
"width": 1080,
"height": 1920,
}
]
),
)
tests/unit/admin_console/test_media_recording_recovery.py
+173
View File
@@ -0,0 +1,173 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Recording files of cancelled / killed workers must survive and be served whole."""
import os
from pathlib import Path
import time
from unittest.mock import patch
import pytest
from apps.admin_console.services import media_service as media_module
MediaService = media_module.MediaService
@pytest.fixture(autouse=True)
def clean_cache():
MediaService._playable_cache.clear()
yield
MediaService._playable_cache.clear()
def _fake_convert(src: Path, dst: Path) -> Path:
dst.write_bytes(b"mp4:" + src.read_bytes())
return dst
def _age(path: Path, seconds: float) -> None:
stamp = time.time() - seconds
os.utime(path, (stamp, stamp))
def test_live_recording_is_not_converted(tmp_path):
mkv = tmp_path / "recording.mkv"
mkv.write_bytes(b"x" * 10) # fresh mtime: scrcpy is still writing
with patch.object(MediaService, "_convert_to_mp4", side_effect=_fake_convert) as convert:
assert MediaService.ensure_browser_playable_video(mkv) == mkv
convert.assert_not_called()
assert not (tmp_path / "recording.mp4").exists()
def test_stale_partial_mp4_is_reconverted_from_newer_mkv(tmp_path):
mkv = tmp_path / "recording.mkv"
mkv.write_bytes(b"full")
partial = tmp_path / "recording.mp4"
partial.write_bytes(b"partial")
_age(partial, 120)
_age(mkv, 60)
with patch.object(MediaService, "_convert_to_mp4", side_effect=_fake_convert) as convert:
result = MediaService.ensure_browser_playable_video(mkv)
convert.assert_called_once()
assert result == partial
assert partial.read_bytes() == b"mp4:full"
def test_finished_mp4_newer_than_mkv_is_reused(tmp_path):
mkv = tmp_path / "recording.mkv"
mkv.write_bytes(b"full")
mp4 = tmp_path / "recording.mp4"
mp4.write_bytes(b"final")
_age(mkv, 120)
_age(mp4, 60)
with patch.object(MediaService, "_convert_to_mp4", side_effect=_fake_convert) as convert:
result = MediaService.ensure_browser_playable_video(mkv)
convert.assert_not_called()
assert result == mp4
def test_build_video_index_skips_live_recordings(tmp_path, monkeypatch):
traces = tmp_path / "traces"
live = traces / "web_1_live"
live.mkdir(parents=True)
(live / "recording.mkv").write_bytes(b"live")
done = traces / "web_2_done_PASS_2026-09-02T12-00-00"
done.mkdir()
(done / "recording.mp4").write_bytes(b"done")
monkeypatch.setattr(media_module, "TRACES_PATH", traces)
monkeypatch.setattr(media_module, "WORKSPACE_ROOT", tmp_path)
with patch.object(MediaService, "_convert_to_mp4", side_effect=_fake_convert) as convert:
idx = MediaService.build_video_index()
convert.assert_not_called()
assert "web_1_live" not in idx
assert idx["web_2_done"] == idx["web_2_done_PASS_2026-09-02T12-00-00"]
assert idx["web_2_done"].endswith("recording.mp4")
def test_strip_trace_status_suffix_handles_every_terminal_marker():
assert MediaService.strip_trace_status_suffix("web_1_PASS_2026") == "web_1"
assert MediaService.strip_trace_status_suffix("web_1_FAIL_2026") == "web_1"
assert MediaService.strip_trace_status_suffix("web_1_TESTFAIL_2026") == "web_1"
assert MediaService.strip_trace_status_suffix("web_1") == "web_1"
def test_recover_orphaned_recording_remuxes_and_renames(tmp_path, monkeypatch):
traces = tmp_path / "traces"
folder = traces / "web_1788391889_5d5287b9"
folder.mkdir(parents=True)
mkv = folder / "recording.mkv"
mkv.write_bytes(b"full")
partial = folder / "recording.mp4"
partial.write_bytes(b"partial")
_age(partial, 120)
_age(mkv, 60)
monkeypatch.setattr(media_module, "TRACES_PATH", traces)
start = time.mktime(time.strptime("2026-09-02 16:31:32", "%Y-%m-%d %H:%M:%S"))
with patch.object(MediaService, "_convert_to_mp4", side_effect=_fake_convert):
final = MediaService.recover_orphaned_recording(str(mkv), start)
assert final is not None
assert final.parent.name == "web_1788391889_5d5287b9_FAIL_2026-09-02T16-31-32"
assert final.name == "recording.mp4"
assert final.read_bytes() == b"mp4:full"
assert not folder.exists()
assert not (final.parent / "recording.mkv").exists()
def test_recover_orphaned_recording_leaves_live_file_alone(tmp_path, monkeypatch):
traces = tmp_path / "traces"
folder = traces / "web_live"
folder.mkdir(parents=True)
mkv = folder / "recording.mkv"
mkv.write_bytes(b"live")
monkeypatch.setattr(media_module, "TRACES_PATH", traces)
with patch.object(MediaService, "_convert_to_mp4", side_effect=_fake_convert) as convert:
assert MediaService.recover_orphaned_recording(str(mkv), None) is None
convert.assert_not_called()
assert folder.exists()
def test_recover_orphaned_recording_keeps_raw_when_remux_fails(tmp_path, monkeypatch):
traces = tmp_path / "traces"
folder = traces / "web_broken"
folder.mkdir(parents=True)
mkv = folder / "recording.mkv"
mkv.write_bytes(b"corrupt")
_age(mkv, 60)
monkeypatch.setattr(media_module, "TRACES_PATH", traces)
with patch.object(MediaService, "_convert_to_mp4", side_effect=lambda src, dst: src):
assert MediaService.recover_orphaned_recording(str(mkv), None) is None
assert mkv.exists()
assert folder.exists()
def test_recover_orphaned_recording_returns_none_without_files(tmp_path):
missing = tmp_path / "missing" / "recording.mkv"
assert MediaService.recover_orphaned_recording(str(missing)) is None
assert MediaService.recover_orphaned_recording(None) is None
tests/unit/admin_console/test_security_boundary.py
+2 -6
View File
@@ -49,9 +49,7 @@ async def test_credentials_endpoint_never_returns_key_material(monkeypatch):
from artemis.config import settings
honeytoken = f"sk-honeytoken-{py_secrets.token_hex(16)}"
monkeypatch.setattr(
type(settings), "get_api_key", lambda self, provider: SecretStr(honeytoken)
)
monkeypatch.setattr(type(settings), "get_api_key", lambda self, provider: SecretStr(honeytoken))
async with _client() as ac:
res = await ac.get("/api/system/credentials")
@@ -168,9 +166,7 @@ async def test_cross_origin_browser_request_is_rejected_without_cors_grant():
@pytest.mark.asyncio
async def test_same_origin_browser_request_passes():
async with _client() as ac:
res = await ac.get(
"/api/system/emulator/status", headers={"Origin": "http://localhost"}
)
res = await ac.get("/api/system/emulator/status", headers={"Origin": "http://localhost"})
assert res.status_code == 200
tests/unit/admin_console/test_server_lifecycle.py
+3 -2
View File
@@ -68,7 +68,9 @@ async def test_stream_events_active_session_replay():
state.current_goal = "Replay Goal"
state.current_profile = "flash"
with patch("apps.admin_console.database.repositories.step_repository.step_repo") as mock_step_repo:
with patch(
"apps.admin_console.database.repositories.step_repository.step_repo"
) as mock_step_repo:
mock_step_repo.get_session_steps.return_value = [
{"step_number": 1, "description": "Step 1", "status": "completed"}
]
@@ -85,7 +87,6 @@ async def test_stream_events_active_session_replay():
await iterator.aclose()
@pytest.mark.asyncio
async def test_windows_sigint_is_ignored_while_task_is_active():
state.current_process = MagicMock(returncode=None)
tests/unit/admin_console/test_session_checks_endpoint.py
+68
View File
@@ -0,0 +1,68 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""``/api/sessions/{id}/checks``: backfill source for the Checker panel."""
import json
from apps.admin_console.services import media_service as media_service_module
from apps.admin_console.services.media_service import MediaService
def test_get_session_checks_reads_ledger_and_outcome(tmp_path, monkeypatch):
monkeypatch.setattr(media_service_module, "TRACES_PATH", tmp_path)
session_dir = tmp_path / "sess-1"
session_dir.mkdir()
records = [
{
"attempt_id": "abc#1",
"checkpoint_id": "abc",
"subgoal_text": "Create the alarm",
"item_text": "alarm exists",
"kind": "verify",
"when": "on_complete",
"status": "passed",
"evidence": "seen",
"ts": 1.0,
},
{"attempt_id": "final#1", "checkpoint_id": "final", "status": "passed", "ts": 2.0},
]
(session_dir / "check_ledger.jsonl").write_text(
"\n".join(json.dumps(r) for r in records) + "\nnot json\n\n", encoding="utf-8"
)
(session_dir / "run_outcome.json").write_text(
json.dumps({"task_status": "completed", "tests": {"passed": 2, "failed": 0}}),
encoding="utf-8",
)
result = MediaService.get_session_checks("sess-1")
assert [r["attempt_id"] for r in result["records"]] == ["abc#1", "final#1"]
assert result["records"][0]["subgoal_text"] == "Create the alarm"
assert result["run_outcome"]["task_status"] == "completed"
def test_get_session_checks_without_material(tmp_path, monkeypatch):
monkeypatch.setattr(media_service_module, "TRACES_PATH", tmp_path)
(tmp_path / "sess-2").mkdir()
assert MediaService.get_session_checks("sess-2") == {"records": [], "run_outcome": None}
assert MediaService.get_session_checks("missing") == {"records": [], "run_outcome": None}
def test_checks_route_is_registered():
from apps.admin_console.routers import media
paths = {getattr(r, "path", None) for r in media.router.routes}
assert "/api/sessions/{session_id}/checks" in paths
tests/unit/admin_console/test_task_device_readiness.py
+5 -16
View File
@@ -40,9 +40,7 @@ async def test_run_task_rejects_locked_device(monkeypatch):
)
run_probe = AsyncMock(return_value=locked_probe)
enqueue_tasks = AsyncMock()
monkeypatch.setattr(
tasks.readiness_engine, "run_device_submission_probe", run_probe
)
monkeypatch.setattr(tasks.readiness_engine, "run_device_submission_probe", run_probe)
monkeypatch.setattr(tasks.task_queue_service, "enqueue_tasks", enqueue_tasks)
with pytest.raises(HTTPException) as exc_info:
@@ -66,9 +64,7 @@ async def test_run_task_enqueues_when_device_is_unlocked(monkeypatch):
)
run_probe = AsyncMock(return_value=unlocked_probe)
enqueue_tasks = AsyncMock(return_value={"status": "started", "tasks": []})
monkeypatch.setattr(
tasks.readiness_engine, "run_device_submission_probe", run_probe
)
monkeypatch.setattr(tasks.readiness_engine, "run_device_submission_probe", run_probe)
monkeypatch.setattr(tasks.task_queue_service, "enqueue_tasks", enqueue_tasks)
result = await tasks.run_task(RunRequest(goal="Open Settings"))
@@ -96,9 +92,7 @@ async def test_run_task_binds_probe_verified_device_when_no_serial_requested(mon
)
run_probe = AsyncMock(return_value=unlocked_probe)
enqueue_tasks = AsyncMock(return_value={"status": "started", "tasks": []})
monkeypatch.setattr(
tasks.readiness_engine, "run_device_submission_probe", run_probe
)
monkeypatch.setattr(tasks.readiness_engine, "run_device_submission_probe", run_probe)
monkeypatch.setattr(tasks.task_queue_service, "enqueue_tasks", enqueue_tasks)
result = await tasks.run_task(RunRequest(goal="Run on any ready device"))
@@ -113,9 +107,7 @@ async def test_run_task_binds_probe_verified_device_when_no_serial_requested(mon
@pytest.mark.asyncio
async def test_idempotent_retry_skips_device_probe_for_active_session(monkeypatch):
run_probe = AsyncMock()
monkeypatch.setattr(
tasks.readiness_engine, "run_device_submission_probe", run_probe
)
monkeypatch.setattr(tasks.readiness_engine, "run_device_submission_probe", run_probe)
monkeypatch.setattr(tasks.state, "active_session_id", "sdk-task-1")
monkeypatch.setattr(tasks.state, "active_connections", {})
monkeypatch.setattr(
@@ -204,9 +196,7 @@ async def test_explicit_device_is_rejected_when_attached_but_not_ready(monkeypat
@pytest.mark.parametrize("enumeration", [None, []])
@pytest.mark.asyncio
async def test_explicit_device_proceeds_when_enumeration_is_indeterminate(
monkeypatch, enumeration
):
async def test_explicit_device_proceeds_when_enumeration_is_indeterminate(monkeypatch, enumeration):
"""A failed (None) or empty enumeration must never hard-reject an explicit
serial: the submission queues and fails downstream if the device is truly
absent. This is the startup-storm regression guard."""
@@ -236,4 +226,3 @@ async def test_explicit_device_proceeds_when_enumeration_is_indeterminate(
enqueue_tasks.assert_awaited_once()
_, kwargs = enqueue_tasks.call_args
assert kwargs.get("device_serial") == "pixel-10"
tests/unit/admin_console/test_task_queue_service.py
+54 -21
View File
@@ -45,6 +45,13 @@ def clean_state(tmp_path, monkeypatch):
queue_module = importlib.import_module("apps.admin_console.services.task_queue_service")
monkeypatch.setattr(state_module, "PAUSE_FILE", isolated_pause_file)
monkeypatch.setattr(queue_module, "PAUSE_FILE", isolated_pause_file)
# The stop assertions below describe the legacy immediate kill; the graceful
# cancel path has its own tests in test_graceful_stop_and_recovery.py.
monkeypatch.setenv("ARTEMIS_CANCEL_GRACE_SECONDS", "0")
monkeypatch.setattr(
"artemis.runtime.cancel_requests.get_temp_dir",
lambda _subfolder=None: isolated_lock_dir,
)
state.clear_queue()
state.queue_items.clear()
state.current_process = None
@@ -283,6 +290,8 @@ async def test_queue_worker_cmd_construction():
enable_outputter=True,
locked_app_package="com.google.android.apps.maps",
app_path="/path/to/app.apk",
verification_level=" Checkpoints ",
explorer_mode="ULTRA",
)
for _ in range(30):
@@ -301,6 +310,12 @@ async def test_queue_worker_cmd_construction():
assert "com.google.android.apps.maps" in cmd
assert "--app-path" in cmd
assert "/path/to/app.apk" in cmd
# Pro tuning knobs are normalised (trimmed, lower-cased) and forwarded
# to the worker as the CLI's existing spelling.
assert cmd[cmd.index("--verification-level") + 1] == "checkpoints"
assert cmd[cmd.index("--explorer-pro-mode") + 1] == "ultra"
assert enqueue_result["tasks"][0]["verification_level"] == "checkpoints"
assert enqueue_result["tasks"][0]["explorer_mode"] == "ultra"
assert (
executed_kwargs[0]["env"]["ARTEMIS_DEVICE_QUEUE_TICKET"]
== (enqueue_result["tasks"][0]["queue_ticket"])
@@ -430,9 +445,7 @@ def test_stop_tasks_terminates_external_global_owner_and_preserves_local_waiter(
patch(
"apps.admin_console.services.task_queue_service.session_repo.update_session_status"
) as update_status,
patch(
"artemis.runtime.trace_store.update_trace_status"
) as update_trace_status,
patch("artemis.runtime.trace_store.update_trace_status") as update_trace_status,
):
assert task_queue_service.stop_tasks(clear_all=False) is True
@@ -510,9 +523,7 @@ def test_stop_tasks_session_target_never_adopts_mismatched_fallback_owner():
patch(
"apps.admin_console.services.task_queue_service.process_supervisor.terminate_tree_verified"
) as terminate_verified,
patch(
"apps.admin_console.services.task_queue_service.session_repo.update_session_status"
),
patch("apps.admin_console.services.task_queue_service.session_repo.update_session_status"),
patch(
"apps.admin_console.services.task_queue_service.session_repo.get_session_by_id",
return_value=None,
@@ -733,7 +744,10 @@ async def test_enqueue_tasks_unified_ingress():
"""Verify enqueue_tasks correctly propagates ingress, custom session_id, and conversation_id."""
with (
patch.object(TaskQueueService, "ensure_worker_running"),
patch("apps.admin_console.services.task_queue_service.DeviceExecutionLock.reserve", return_value="mock-ticket-unified"),
patch(
"apps.admin_console.services.task_queue_service.DeviceExecutionLock.reserve",
return_value="mock-ticket-unified",
),
):
res = await task_queue_service.enqueue_tasks(
["Test unified goal"],
@@ -885,7 +899,9 @@ def test_stop_tasks_by_device_id_targets_specific_device():
"apps.admin_console.services.task_queue_service.DeviceExecutionLock.is_active_owner",
return_value=True,
),
patch("apps.admin_console.services.task_queue_service.DeviceExecutionLock.cleanup_stale_locks"),
patch(
"apps.admin_console.services.task_queue_service.DeviceExecutionLock.cleanup_stale_locks"
),
patch(
"apps.admin_console.services.task_queue_service.process_supervisor.terminate_tree_verified",
return_value=True,
@@ -899,8 +915,18 @@ def test_stop_tasks_by_device_id_targets_specific_device():
def test_stop_tasks_queued_item_by_session_id():
"""Stopping a queued task by session_id cancels its reservation and removes it from queue."""
state.queue_items = [
{"session_id": "queue-1", "goal": "Goal 1", "status": "pending", "queue_ticket": "ticket-1"},
{"session_id": "queue-2", "goal": "Goal 2", "status": "pending", "queue_ticket": "ticket-2"},
{
"session_id": "queue-1",
"goal": "Goal 1",
"status": "pending",
"queue_ticket": "ticket-1",
},
{
"session_id": "queue-2",
"goal": "Goal 2",
"status": "pending",
"queue_ticket": "ticket-2",
},
]
with (
@@ -916,7 +942,9 @@ def test_stop_tasks_queued_item_by_session_id():
"apps.admin_console.services.task_queue_service.DeviceExecutionLock.cancel_reservation",
return_value=True,
) as cancel_res,
patch("apps.admin_console.services.task_queue_service.session_repo.update_session_status") as update_status,
patch(
"apps.admin_console.services.task_queue_service.session_repo.update_session_status"
) as update_status,
):
assert task_queue_service.stop_tasks(clear_all=False, session_id="queue-1") is True
cancel_res.assert_called_once_with("ticket-1")
@@ -934,7 +962,10 @@ async def test_enqueue_tasks_deduplicates_by_session_id():
with (
patch("apps.admin_console.services.task_queue_service.session_repo"),
patch("apps.admin_console.services.task_queue_service.DeviceExecutionLock.reserve", return_value="ticket-123"),
patch(
"apps.admin_console.services.task_queue_service.DeviceExecutionLock.reserve",
return_value="ticket-123",
),
):
res1 = await task_queue_service.enqueue_tasks(["Task goal"], session_id="sid-dedup-1")
assert len(state.queue_items) == 1
@@ -1041,8 +1072,7 @@ async def test_manual_stop_of_one_run_does_not_pollute_concurrent_run():
assert ended_payloads["run-b"]["was_stopped_manually"] is False
persisted = {
call.args[0]: call.args[1]
for call in mock_repo.update_session_status.call_args_list
call.args[0]: call.args[1] for call in mock_repo.update_session_status.call_args_list
}
assert persisted.get("run-a") == "cancelled"
assert persisted.get("run-b") == "completed"
@@ -1059,22 +1089,25 @@ async def test_enqueue_tasks_debounces_rapid_identical_submissions():
with (
patch("apps.admin_console.services.task_queue_service.session_repo"),
patch("apps.admin_console.services.task_queue_service.DeviceExecutionLock.reserve", return_value="ticket-456"),
patch(
"apps.admin_console.services.task_queue_service.DeviceExecutionLock.reserve",
return_value="ticket-456",
),
patch(
"artemis.runtime.device_pool.device_pool.try_list_devices_async",
new=AsyncMock(return_value=[]),
),
patch.object(TaskQueueService, "ensure_worker_running"),
):
res1 = await task_queue_service.enqueue_tasks(["Rapid duplicate goal"], device_serial="dev-1")
res1 = await task_queue_service.enqueue_tasks(
["Rapid duplicate goal"], device_serial="dev-1"
)
assert len(state.queue_items) == 1
assert res1["enqueued_count"] == 1
# Immediate second submission (within 1s) must debounce
res2 = await task_queue_service.enqueue_tasks(["Rapid duplicate goal"], device_serial="dev-1")
res2 = await task_queue_service.enqueue_tasks(
["Rapid duplicate goal"], device_serial="dev-1"
)
assert len(state.queue_items) == 1
assert res2["enqueued_count"] == 0
tests/unit/agents/test_check_prompt_assembly.py
+7 -5
View File
@@ -67,7 +67,9 @@ def test_planner_blocks_with_checks_mount_generation_and_audit():
assert "Never silently promise" in generation
audit = data["blocks"]["check_audit"]
assert "REJECT" in audit
# Advisory review: the audit flags weakened check standards, never blocks.
assert "FLAG" in audit
assert "REJECT" not in audit
def _mock_ctx(tmp_path=None, plan: str | None = None, **setup_kwargs):
@@ -108,7 +110,7 @@ async def test_explainer_renders_iff_plan_has_check_lines(tmp_path):
await component(builder2, _state(), ctx2)
joined = "\n".join(p for p in builder2.human_parts if isinstance(p, str))
assert "check lines" in joined
assert "independent Checker" in joined
assert "never declare a check passed yourself" in joined
assert "automatically restored" in joined
@@ -148,13 +150,13 @@ async def test_main_template_diagnosis_trigger_gated_on_verification():
with_checks = await _render_main_template(
disable_checker=False, disable_planner_validation=True
)
assert "Ambiguous Validation Rejection or Verification Finding" in with_checks
assert "Ambiguous Verification Finding" in with_checks
# Planner validation alone can also produce rejections.
# Planner validation alone can also produce (advisory) findings.
with_validation = await _render_main_template(
disable_checker=True, disable_planner_validation=False
)
assert "Ambiguous Validation Rejection or Verification Finding" in with_validation
assert "Ambiguous Verification Finding" in with_validation
@pytest.mark.asyncio
tests/unit/agents/test_checker_structured_report.py
+55
View File
@@ -0,0 +1,55 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The structured verdict request must never end on a model turn (Gemini
rejects such requests with 400 "Requests ending with a model turn are not
supported"), which is the natural state after the loop's tool-free reply."""
from unittest.mock import AsyncMock, MagicMock
import pytest
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
from artemis.agents.checker.checker import CheckReport, _structured_report
def _llm(report: CheckReport):
structured = MagicMock()
structured.ainvoke = AsyncMock(return_value=report)
llm = MagicMock()
llm.with_structured_output.return_value = structured
return llm, structured
@pytest.mark.asyncio
async def test_structured_report_appends_user_turn_after_model_reply():
llm, structured = _llm(CheckReport(verdicts=[]))
messages = [SystemMessage(content="s"), HumanMessage(content="h"), AIMessage(content="done")]
await _structured_report(llm, messages)
sent = structured.ainvoke.call_args.args[0]
assert isinstance(sent[-1], HumanMessage)
assert sent[-2] is messages[-1]
assert len(messages) == 3 # caller's list untouched
@pytest.mark.asyncio
async def test_structured_report_keeps_user_ending_conversation_as_is():
llm, structured = _llm(CheckReport(verdicts=[]))
messages = [SystemMessage(content="s"), HumanMessage(content="final iteration")]
await _structured_report(llm, messages)
assert structured.ainvoke.call_args.args[0] is messages
tests/unit/agents/test_execution_incident_prompt.py
+197
View File
@@ -0,0 +1,197 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The Operator-facing execution incident block and the incident record."""
from types import SimpleNamespace
import pytest
from artemis.agents.operator.prompts import (
EXECUTION_INCIDENT_MARKER,
ExecutionIncidentPromptComponent,
PromptBuilder,
render_closed_incident,
render_execution_incident,
)
from artemis.agents.validator.categories import ValidationErrorCategory
from artemis.agents.validator.incidents import (
KIND_EXEC_ERROR,
KIND_SAFETY_NET,
ExecutionIncident,
open_incident,
)
STEPS = [
{
"step_number": 11,
"action_taken": [{"action": "click", "coordinates": [540, 1800], "target_text": "player"}],
"last_execution_result": {"status": "success"},
},
{
"step_number": 12,
"action_taken": [{"action": "click", "target_text": "Skip"}],
"last_execution_result": {"status": "failed"},
},
]
def _disappeared(consecutive=1, **overrides):
incident = open_incident(
previous=None,
kind=KIND_SAFETY_NET,
category=ValidationErrorCategory.TARGET_DISAPPEARED,
reason="Target element 'Skip' was not found on the screen.",
action_item={
"action": "tap",
"coordinates": [950, 288],
"normalized_coordinates": [880, 120],
"target_text": "Skip",
"attempts": ["Pre-execution validation failed: ..."],
},
action_description="Tapped 'Skip' at [950, 288]",
step_number=12,
)
incident["consecutive_failures"] = consecutive
incident.update(overrides)
return incident
# --- record -------------------------------------------------------------------------
def test_open_incident_strips_internal_keys_and_starts_at_one():
incident = _disappeared()
assert incident["consecutive_failures"] == 1
assert "attempts" not in incident["action"]
assert incident["category"] == "target_disappeared"
assert incident["burst_size"] == 1 and incident["action_index"] == 0
def test_open_incident_continues_the_previous_count():
previous = _disappeared(consecutive=2)
incident = open_incident(
previous=previous,
kind=KIND_EXEC_ERROR,
category=ValidationErrorCategory.GENERAL,
reason="Error: tap rejected",
action_item={"action": "tap"},
action_description="Tapped element",
)
assert incident["consecutive_failures"] == 3
assert ExecutionIncident.from_dict(incident).is_burst is False
assert ExecutionIncident.from_dict(None) is None
assert ExecutionIncident.from_dict({"kind": "safety_net", "unknown_key": 1}) is not None
# --- rendering ----------------------------------------------------------------------
def test_render_disappeared_names_target_trigger_and_burst_exception():
text = render_execution_incident(_disappeared(), STEPS)
assert text.startswith(EXECUTION_INCIDENT_MARKER)
assert "Opened at Step 12; consecutive failed turns: 1." in text
assert "was NOT executed. The pre-execution safety net refused it" in text
# Recorded target with normalized coordinates and text.
assert 'normalized [880, 120] "Skip"' in text
# The last successful action before the incident is the trigger candidate.
assert "Your last successfully executed action was `Tapped 'player' at [540, 1800]`" in text
assert "(Step 11)" in text
# Facts and evidence only: the response protocol is stated once, in the
# static system prompt, so the per-turn block never restates it.
assert "How to respond" not in text
assert "settle the original intent" not in text
assert "e.g." not in text
# No escalation ladder: the Operator keeps its freedom.
assert "Escalation" not in text
assert "ask_diagnoser" not in text
def test_render_shifted_and_occupied_evidence():
shifted = _disappeared(
category="target_shifted",
evidence={"new_location": [500, 640], "new_bounds": [400, 600, 600, 680]},
)
text = render_execution_incident(shifted, STEPS)
assert "has moved at normalized [500, 640] (bounds [400, 600, 600, 680])" in text
occupied = _disappeared(category="target_occupied", evidence={"occupant": "'Sign Up' button"})
text = render_execution_incident(occupied, STEPS)
assert "now covered by 'Sign Up' button" in text
def test_render_burst_abort_and_exec_error():
burst = _disappeared(kind=KIND_EXEC_ERROR, category="general", burst_size=3, action_index=1)
burst["reason"] = "Error: tap rejected"
text = render_execution_incident(burst, STEPS)
assert "action 2 of your 3-action fast burst" in text
assert "The 1 action(s) after it were NOT executed" in text
plain = _disappeared(kind=KIND_EXEC_ERROR, category="general")
text = render_execution_incident(plain, STEPS)
assert "was dispatched, but the device/executor reported" in text
def test_render_without_history_omits_the_trigger_hint():
text = render_execution_incident(_disappeared(), [])
assert "last successfully executed action" not in text
assert 'Its recorded target was normalized [880, 120] "Skip".' in text
# --- component ----------------------------------------------------------------------
@pytest.mark.asyncio
async def test_component_renders_only_while_an_incident_is_open():
builder = PromptBuilder()
ctx = SimpleNamespace(data_engine=None)
await ExecutionIncidentPromptComponent()(
builder, SimpleNamespace(open_incident=None), ctx, steps=STEPS
)
assert builder.human_parts == []
await ExecutionIncidentPromptComponent()(
builder, SimpleNamespace(open_incident=_disappeared()), ctx, steps=STEPS
)
assert len(builder.human_parts) == 1
# Its own text part: the transcript scrub keeps it verbatim across turns.
assert builder.human_parts[0].startswith(EXECUTION_INCIDENT_MARKER)
def test_render_closed_notice_settles_the_intent():
closed = dict(_disappeared(), closed_at_step=15)
text = render_closed_incident(closed)
assert text.startswith("--- Execution Incident (CLOSED at Step 15) ---")
assert "opened at Step 12 on `Tapped 'Skip' at [950, 288]`" in text
assert "already served, still pending, or no longer needed" in text
@pytest.mark.asyncio
async def test_component_renders_closed_notice_once_no_incident_is_open():
builder = PromptBuilder()
ctx = SimpleNamespace(data_engine=None)
state = SimpleNamespace(
open_incident=None, last_closed_incident=dict(_disappeared(), closed_at_step=15)
)
await ExecutionIncidentPromptComponent()(builder, state, ctx, steps=STEPS)
assert len(builder.human_parts) == 1
assert builder.human_parts[0].startswith("--- Execution Incident (CLOSED at Step 15) ---")
# An open incident takes precedence over a stale closed notice.
builder = PromptBuilder()
state = SimpleNamespace(
open_incident=_disappeared(), last_closed_incident=dict(_disappeared(), closed_at_step=15)
)
await ExecutionIncidentPromptComponent()(builder, state, ctx, steps=STEPS)
assert builder.human_parts[0].startswith(EXECUTION_INCIDENT_MARKER)
tests/unit/agents/test_explorer.py
+67 -74
View File
@@ -1041,25 +1041,26 @@ async def test_explorer_inspect_region():
@pytest.mark.asyncio
async def test_explorer_final_turn_tool_stripping():
"""Test that on the final iteration (turn 10), Explorer strips other tools and injects warning."""
"""On the final turn the Explorer strips other tools and injects the warning."""
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.device = MagicMock()
mock_ctx.device.device_width = 1080
mock_ctx.device.device_height = 2400
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = "/tmp/test_session"
explorer = Explorer(mock_ctx)
mock_ctx.llm_config = MagicMock()
mock_ctx.llm_config.explorer = MagicMock()
mock_ctx.llm_config.explorer.model = "gemini-3.7-flash"
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test_screenshot.jpg"
# Set up generate_content mock to capture call arguments
# A client already shared on the context selects the native engine and is
# reused lazily by ``run`` (no ``genai.Client`` construction).
mock_client = MagicMock()
mock_ctx._genai_client = mock_client
explorer = Explorer(mock_ctx)
assert explorer.use_native_gemini is True
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test_screenshot.jpg"
# Turn 1: model returns search_xml_ocr
fc_search = MagicMock()
@@ -1094,14 +1095,14 @@ async def test_explorer_final_turn_tool_stripping():
minimal_list="test list",
)
# Total number of iterations is max_iterations = 3 under "pro" mode default
# The default "pro" tier budgets 3 turns (tiers.EXPLORER_TIERS)
assert mock_client.aio.models.generate_content.call_count == 3
# Check the final turn call (index 2) keyword args
call_args_3 = mock_client.aio.models.generate_content.call_args_list[2]
config = call_args_3.kwargs["config"]
# Tools in the 10th call should only contain "submit_answer"
# Tools in the final call should only contain "submit_answer"
tools = config.tools[0].function_declarations
assert len(tools) == 1
assert tools[0].name == "submit_answer"
@@ -1120,9 +1121,16 @@ async def test_explorer_final_turn_tool_stripping():
@pytest.mark.asyncio
async def test_explorer_ask_perception_tool():
"""The perception bundle answers from the in-memory screen index.
No Data Engine lookup is involved: the text search, the coordinate
audit and the OCR inventory all read ``explorer.screen_index``.
"""
import numpy as np
from unittest.mock import mock_open
from artemis.agents.explorer.screen_index import ScreenIndex
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = "/tmp/test_session"
@@ -1144,37 +1152,24 @@ async def test_explorer_ask_perception_tool():
"description": "Original complete screenshot",
}
}
# Mock search_ui_func
mock_search_ui_res = {
"matches": [
explorer.screen_index = ScreenIndex.from_hierarchy(
[
{
"matched_text": "Settings",
"bounds": [540, 480, 540, 480],
"type": "xml",
"bounds": "[500,400][600,560]",
"text": "Settings",
"resource-id": "com.android.settings:id/btn_settings",
"class": "android.widget.Button",
"clickable": "true",
},
{
"matched_text": "Sign In",
"bounds": [100, 100, 200, 200],
"type": "ocr",
"bounds": "[100,100][200,200]",
"class": "android.widget.FrameLayout [OCR]",
"ocr_elements": [{"text": "Settings page", "bounds": "[100,100][200,200]"}],
},
]
}
# Mock storage record
mock_record = MagicMock()
mock_record.ui_tree = [
{
"bounds": "[500,400][600,560]",
"text": "Settings Button",
"resource-id": "com.android.settings:id/btn_settings",
"class": "android.widget.Button",
}
]
mock_record.ocr_result = []
mock_storage = MagicMock()
mock_storage.get_image.return_value = mock_record
],
1080,
2400,
)
# Mock _run_object_detection
mock_detector_res = {
@@ -1186,18 +1181,7 @@ async def test_explorer_ask_perception_tool():
mock_draw_dots = MagicMock()
with (
patch(
"artemis.agents.explorer.explorer.search_ui_func",
return_value=mock_search_ui_res,
),
patch(
"artemis.agents.explorer.explorer.StorageManager",
return_value=mock_storage,
),
patch(
"artemis.mcp.xml_search_server.StorageManager",
return_value=mock_storage,
),
patch("artemis.agents.explorer.explorer.StorageManager") as storage_cls,
patch(
"artemis.agents.explorer.explorer._run_object_detection",
return_value=mock_detector_res,
@@ -1216,37 +1200,46 @@ async def test_explorer_ask_perception_tool():
detect_queries=["profile icon"],
)
assert "Text Search Results are:" in result["text"]
assert "Coordinate Search Results are:" in result["text"]
assert "Object Detection Results are:" in result["text"]
storage_cls.assert_not_called()
assert "Text Search Results are:" in result["text"]
assert "Coordinate Search Results are:" in result["text"]
assert "Object Detection Results are:" in result["text"]
# XML Text search should return label X1
assert "[X1]" in result["text"]
# OCR Text search should return label O2
assert "[O2]" in result["text"]
# Coordinate search should return label X3
assert "[X3]" in result["text"]
# Object detection should return label D4 (since global label idx is shared/contiguous)
assert "[D4]" in result["text"]
# UI-tree text match -> X1, OCR text match -> O2 (both from the index)
assert "[X1] 'Settings' at [509,200]" in result["text"]
assert "[O2] 'Settings page' at [138,62]" in result["text"]
# Coordinate audit -> X3 with the innermost element's description
assert (
"Matched element at [550,200]: [X3] | 'Settings' class=android.widget.Button"
" id=com.android.settings:id/btn_settings source=ui-tree interactive"
) in result["text"]
# Object detection -> D4 (the label counter is shared and contiguous)
assert "[D4]" in result["text"]
assert len(result["image_paths"]) == 3
# Element-backed labels are registered; detection points are not.
assert explorer.label_registry["X1"].text == "Settings"
assert explorer.label_registry["O2"].source == "ocr"
assert explorer.label_registry["X3"] is explorer.label_registry["X1"]
assert "D4" not in explorer.label_registry
# Verify call arguments of draw_dots
assert mock_draw_dots.call_count == 3
# First call (XML/OCR Text search): green color
first_call = mock_draw_dots.call_args_list[0]
assert first_call.kwargs.get("color") == "green"
assert first_call.args[2] == ["X1", "O2"]
assert len(result["image_paths"]) == 3
# Second call (XML Coordinate audit): blue color
second_call = mock_draw_dots.call_args_list[1]
assert second_call.kwargs.get("color") == "blue"
assert second_call.args[2] == ["X3"]
# Verify call arguments of draw_dots
assert mock_draw_dots.call_count == 3
# First call (XML/OCR Text search): green color
first_call = mock_draw_dots.call_args_list[0]
assert first_call.kwargs.get("color") == "green"
assert first_call.args[2] == ["X1", "O2"]
# Third call (Object detection): red color
third_call = mock_draw_dots.call_args_list[2]
assert third_call.kwargs.get("color") == "red"
assert third_call.args[2] == ["D4"]
# Second call (XML Coordinate audit): blue color
second_call = mock_draw_dots.call_args_list[1]
assert second_call.kwargs.get("color") == "blue"
assert second_call.args[2] == ["X3"]
# Third call (Object detection): red color
third_call = mock_draw_dots.call_args_list[2]
assert third_call.kwargs.get("color") == "red"
assert third_call.args[2] == ["D4"]
def test_explorer_prune_historical_images():
tests/unit/agents/test_explorer_geometry.py
+150
View File
@@ -0,0 +1,150 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Screen geometry shared by the Explorer and the ``ask_explorer`` tool."""
from types import SimpleNamespace
import pytest
from artemis.agents.explorer.geometry import (
FALLBACK_SCREEN_SIZE,
NORMALIZED_MAX,
is_valid_norm_point,
norm_to_pixel,
pixel_to_norm,
resolve_screen_size,
)
def _ctx(width=None, height=None, device=True):
if not device:
return SimpleNamespace(device=None)
return SimpleNamespace(device=SimpleNamespace(device_width=width, device_height=height))
def _state(raw):
return SimpleNamespace(operator_raw_data=raw)
# ---------------------------------------------------------------------------
# resolve_screen_size
# ---------------------------------------------------------------------------
def test_operator_observation_wins_over_the_device_context():
ctx = _ctx(1080, 2400)
state = _state({"width": 720, "height": 1600})
assert resolve_screen_size(ctx, state) == (720, 1600)
def test_device_context_is_used_when_the_observation_has_no_size():
ctx = _ctx(1080, 2400)
assert resolve_screen_size(ctx, _state(None)) == (1080, 2400)
assert resolve_screen_size(ctx, _state({})) == (1080, 2400)
assert resolve_screen_size(ctx, _state("not a dict")) == (1080, 2400)
@pytest.mark.parametrize(
"raw",
[
{"width": 0, "height": 1600},
{"width": 720, "height": -1},
{"width": "720", "height": 1600},
{"width": 720.0, "height": 1600},
{"width": None, "height": None},
{"width": 720},
],
)
def test_non_int_or_non_positive_observation_sizes_are_ignored(raw):
ctx = _ctx(1080, 2400)
assert resolve_screen_size(ctx, _state(raw)) == (1080, 2400)
def test_fallback_when_neither_source_reports_a_size():
assert resolve_screen_size(_ctx(device=False), _state(None)) == FALLBACK_SCREEN_SIZE
assert resolve_screen_size(_ctx(None, None), _state(None)) == FALLBACK_SCREEN_SIZE
assert resolve_screen_size(SimpleNamespace(), SimpleNamespace()) == FALLBACK_SCREEN_SIZE
def test_device_dimensions_fall_back_independently():
fb_w, fb_h = FALLBACK_SCREEN_SIZE
assert resolve_screen_size(_ctx(1440, 0), _state(None)) == (1440, fb_h)
assert resolve_screen_size(_ctx("1440", 3200), _state(None)) == (fb_w, 3200)
assert resolve_screen_size(_ctx(None, 3200), _state(None)) == (fb_w, 3200)
# ---------------------------------------------------------------------------
# norm_to_pixel / pixel_to_norm
# ---------------------------------------------------------------------------
def test_norm_to_pixel_scales_on_the_0_1000_grid():
assert norm_to_pixel(500, 500, 1080, 2400) == (540, 1200)
assert norm_to_pixel(0, 0, 1080, 2400) == (0, 0)
assert norm_to_pixel(NORMALIZED_MAX, NORMALIZED_MAX, 1080, 2400) == (1080, 2400)
assert norm_to_pixel(250, 750, 1080, 2400) == (270, 1800)
def test_norm_to_pixel_clamps_out_of_range_inputs():
assert norm_to_pixel(-50, -1, 1080, 2400) == (0, 0)
assert norm_to_pixel(1500, 2000, 1080, 2400) == (1080, 2400)
def test_pixel_to_norm_scales_back_to_the_0_1000_grid():
assert pixel_to_norm(540, 1200, 1080, 2400) == (500, 500)
assert pixel_to_norm(0, 0, 1080, 2400) == (0, 0)
assert pixel_to_norm(1080, 2400, 1080, 2400) == (NORMALIZED_MAX, NORMALIZED_MAX)
def test_pixel_to_norm_clamps_out_of_range_inputs():
assert pixel_to_norm(-10, -10, 1080, 2400) == (0, 0)
assert pixel_to_norm(5000, 9000, 1080, 2400) == (NORMALIZED_MAX, NORMALIZED_MAX)
def test_conversions_round_trip_on_grid_points():
for nx, ny in [(0, 0), (500, 500), (1000, 1000), (250, 750)]:
px, py = norm_to_pixel(nx, ny, 1000, 2000)
assert pixel_to_norm(px, py, 1000, 2000) == (nx, ny)
# ---------------------------------------------------------------------------
# is_valid_norm_point
# ---------------------------------------------------------------------------
@pytest.mark.parametrize(
"coords",
[[0, 0], [500, 500], (1000, 1000), ["500", "250"], [12.7, 999.2]],
)
def test_valid_norm_points(coords):
assert is_valid_norm_point(coords)
@pytest.mark.parametrize(
"coords",
[
None,
"500,500",
[500],
[500, 500, 500],
[-1, 0],
[0, 1001],
["x", 0],
[None, 5],
{"x": 1, "y": 2},
],
)
def test_invalid_norm_points(coords):
assert not is_valid_norm_point(coords)
tests/unit/agents/test_explorer_perception_regressions.py
+65
View File
@@ -0,0 +1,65 @@
"""Regression tests for Explorer perception-tool failure paths."""
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from artemis.agents.explorer.perception_tools import PerceptionToolsMixin
from artemis.agents.explorer.screen_index import ScreenIndex
def _perception_host() -> PerceptionToolsMixin:
host = PerceptionToolsMixin()
host.ctx = MagicMock()
host.global_label_idx = 1
host.width = 1080
host.height = 2400
host.image_name = "screen-1"
host.screenshot_path = "screen-1.jpg"
host.image_pool = {
"img_0": {
"path": "screen-1.jpg",
"transform": {
"offset_x": 0.0,
"offset_y": 0.0,
"scale_x": 1.0,
"scale_y": 1.0,
},
}
}
host.next_img_id = 1
host.screen_index = ScreenIndex.empty(1080, 2400)
host.label_registry = {}
return host
@pytest.mark.asyncio
async def test_get_ocr_list_reads_the_screen_index_without_storage() -> None:
host = _perception_host()
with patch("artemis.agents.explorer.explorer.StorageManager") as storage_cls:
result = await host.exec_get_ocr_list()
assert result == {
"text": "No text elements detected on the screen.",
"image_path": None,
}
storage_cls.assert_not_called()
@pytest.mark.asyncio
async def test_image_processor_empty_outputs_returns_stable_error() -> None:
host = _perception_host()
processor = MagicMock()
processor.run = AsyncMock(return_value={"outputs": [], "summary": "nothing found"})
with patch(
"artemis.agents.explorer.perception_tools.ImageProcessor",
return_value=processor,
):
result = await host.exec_ask_image_processor("crop", target_image_id="img_0")
assert result == {
"text": "ImageProcessor error: no output images were produced.",
"image_paths": [],
}
tests/unit/agents/test_explorer_prepass.py
+463
View File
@@ -0,0 +1,463 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Explorer in-memory screen search, label registry, bounds enrichment and pre-pass."""
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from artemis.agents.explorer.explorer import Explorer
from artemis.agents.explorer.screen_index import ScreenElement, ScreenIndex
from artemis.context import ArtemisContext
from artemis.graph.state import State
FACADE = "artemis.agents.explorer.explorer"
W, H = 1080, 2400
#: One "Settings" button, two "Send" buttons and two OCR-only texts.
HIERARCHY = [
{
"text": "Settings",
"bounds": "[500,400][600,560]",
"class": "android.widget.Button",
"resource-id": "app:id/settings",
"clickable": "true",
},
{"text": "Send", "bounds": "[100,2000][300,2100]", "class": "android.widget.Button"},
{"text": "Send", "bounds": "[700,2000][900,2100]", "class": "android.widget.Button"},
{
"bounds": "[0,0][1080,2400]",
"class": "android.widget.FrameLayout",
"ocr_elements": [
{"text": "Settings page", "bounds": "[100,100][300,160]"},
{"text": "Welcome", "bounds": "[100,200][400,260]"},
],
},
]
#: "Settings" [500,400][600,560] on a 1080x2400 screen, in normalized units.
SETTINGS_COORDS = [509, 200]
SETTINGS_BOUNDS = [462, 166, 555, 233]
def _context() -> MagicMock:
ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock()
ctx.device.device_width = W
ctx.device.device_height = H
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = None
ctx.llm_config = MagicMock()
ctx.llm_config.explorer = MagicMock()
ctx.llm_config.explorer.model = "gemini-3.8-flash"
ctx.agent_config = MagicMock()
ctx.agent_config.denylisted_tools = {}
return ctx
def _state(screenshot: str, hierarchy=None) -> MagicMock:
state = MagicMock(spec=State)
state.latest_screenshot = screenshot
state.latest_ui_hierarchy = HIERARCHY if hierarchy is None else hierarchy
state.operator_raw_data = {}
return state
def _indexed_explorer(hierarchy=None) -> Explorer:
"""An Explorer whose screen is already indexed (no ``run`` involved)."""
explorer = Explorer(_context())
explorer.width, explorer.height = W, H
explorer.screenshot_path = "/tmp/screen.jpg"
explorer.screen_index = ScreenIndex.from_hierarchy(
HIERARCHY if hierarchy is None else hierarchy, W, H
)
return explorer
def _storage(record=None) -> MagicMock:
storage = MagicMock()
storage.get_image.return_value = record
return storage
def _outcome(candidates=None, fallback: str = "") -> str:
return json.dumps({"candidates": candidates or [], "fallback_message": fallback})
@pytest.fixture
def screenshot(tmp_path):
path = tmp_path / "screen.jpg"
path.write_bytes(b"fake-jpeg-bytes")
return str(path)
@pytest.fixture
def no_drawing():
"""Keeps the annotation helpers away from the file system."""
with (
patch(f"{FACADE}.draw_dots") as draw_dots,
patch("pathlib.Path.mkdir"),
patch("glob.glob", return_value=[]),
):
yield draw_dots
# --------------------------------------------------------------------------- #
# Screen index construction inside run()
# --------------------------------------------------------------------------- #
@pytest.mark.asyncio
async def test_index_is_built_from_state_hierarchy_when_record_is_missing(screenshot, no_drawing):
explorer = Explorer(_context())
explorer._run_flash = AsyncMock(return_value=_outcome(fallback="nothing"))
with (
patch(f"{FACADE}.StorageManager", return_value=_storage(None)),
patch(f"{FACADE}.perform_ocr", new_callable=AsyncMock) as perform_ocr,
):
await explorer.run("gear icon", "", screenshot, _state(screenshot), version="flash")
text = (await explorer._search_ui_helper("Settings", prefix="X"))["text"]
perform_ocr.assert_not_awaited()
assert len(explorer.screen_index) == 5
assert "not found in Data Engine" not in text
assert "[X1] 'Settings' at [509,200]" in text
@pytest.mark.asyncio
async def test_index_ignores_state_hierarchy_for_another_screenshot(screenshot):
explorer = Explorer(_context())
explorer._run_flash = AsyncMock(return_value=_outcome())
state = _state("/tmp/other.jpg")
with patch(f"{FACADE}.StorageManager", return_value=_storage(None)):
await explorer.run("Settings", "", screenshot, state, version="flash")
assert len(explorer.screen_index) == 0
explorer._run_flash.assert_awaited_once_with("Settings", screenshot)
assert (await explorer._search_ui_helper("Settings"))["text"] == (
"No UI-tree data is available for this screen; rely on visual detection."
)
# --------------------------------------------------------------------------- #
# Perception helpers answer from the index
# --------------------------------------------------------------------------- #
@pytest.mark.asyncio
async def test_search_ui_helper_merges_lenient_matches_without_duplicates(no_drawing):
hierarchy = [*HIERARCHY, {"text": "Settled", "bounds": "[100,300][300,360]"}]
explorer = _indexed_explorer(hierarchy)
spy = MagicMock(wraps=explorer.screen_index.search_text)
explorer.screen_index.search_text = spy
result = await explorer._search_ui_helper("Settings", prefix="X", color="green")
assert [call.args[1] for call in spy.call_args_list] == [0.7, 0.4]
parts = result["text"].split(" | ")
assert parts[0] == "[X1] 'Settings' at [509,200]"
assert parts[1] == "[O2] 'Settings page' at [185,54]"
# "Settled" (similarity ~0.53) only arrives through the lenient pass...
assert any(part.startswith("[X") and "'Settled'" in part for part in parts)
# ...and the strict hits are not listed a second time.
assert sum("'Settings'" in part for part in parts) == 1
assert explorer.label_registry["X1"].text == "Settings"
assert explorer.label_registry["O2"].source == "ocr"
assert no_drawing.call_args.kwargs["color"] == "green"
assert result["image_path"] is not None
@pytest.mark.asyncio
async def test_search_ui_helper_skips_lenient_pass_with_enough_strict_hits(no_drawing):
explorer = _indexed_explorer()
strict = explorer.screen_index.search_text("Settings", 0.4) # 5 hits, all "strict"
explorer.screen_index.search_text = MagicMock(return_value=strict)
await explorer._search_ui_helper("Settings", prefix="X")
explorer.screen_index.search_text.assert_called_once_with("Settings", 0.7)
@pytest.mark.asyncio
async def test_search_by_coords_helper_reports_innermost_first(no_drawing):
explorer = _indexed_explorer(
[
{"text": "Card", "bounds": "[0,0][1080,1200]", "class": "android.widget.FrameLayout"},
{"text": "Title", "bounds": "[500,400][600,560]", "class": "android.widget.TextView"},
]
)
hit = await explorer._search_by_coords_helper(550, 200, prefix="X", color="blue")
miss = await explorer._search_by_coords_helper(10, 990, prefix="X", color="blue")
assert hit["text"] == (
"Matched element at [550,200]: [X1]"
" | 'Title' class=android.widget.TextView source=ui-tree"
" | 'Card' class=android.widget.FrameLayout source=ui-tree"
)
assert explorer.label_registry["X1"].text == "Title"
assert miss["text"] == "No elements found at [10,990]."
assert "X2" not in explorer.label_registry
# The blue dot is drawn for both probes at the pixel position.
assert no_drawing.call_count == 2
assert no_drawing.call_args_list[0].args[1] == [[594, 480]]
assert no_drawing.call_args_list[0].args[2] == ["X1"]
@pytest.mark.asyncio
async def test_get_ocr_list_reads_the_index(no_drawing):
explorer = _indexed_explorer()
with patch(f"{FACADE}.StorageManager") as storage_cls:
result = await explorer.exec_get_ocr_list()
storage_cls.assert_not_called()
assert result["text"].splitlines() == [
"[O1] 'Settings page' coords: [185,54]",
"[O2] 'Welcome' coords: [231,95]",
]
assert result["image_path"] is not None
assert {k: v.source for k, v in explorer.label_registry.items()} == {"O1": "ocr", "O2": "ocr"}
@pytest.mark.asyncio
async def test_get_ocr_list_without_ocr_elements():
explorer = _indexed_explorer([{"text": "Only XML", "bounds": "[0,0][100,100]"}])
result = await explorer.exec_get_ocr_list()
assert result == {"text": "No text elements detected on the screen.", "image_path": None}
# --------------------------------------------------------------------------- #
# Label registry from the initial numbered list
# --------------------------------------------------------------------------- #
def test_initial_list_registers_numbered_labels(no_drawing):
explorer = _indexed_explorer()
minimal_list, _marked = explorer._annotate_initial_screenshot(
HIERARCHY, "/tmp/screen.jpg", None, ""
)
assert minimal_list.splitlines()[0].startswith("[1] Text: 'Settings'")
assert sorted(explorer.label_registry) == ["1", "2", "3", "4", "5"]
settings = explorer.label_registry["1"]
assert settings.bounds == (500, 400, 600, 560)
assert settings.source == "xml" and settings.interactive # taken from the index
assert explorer.label_registry["4"].source == "ocr"
assert explorer.label_registry["4"].text == "Settings page"
assert explorer.global_label_idx == 6
assert no_drawing.call_args.args[1][0] == [550, 480]
# --------------------------------------------------------------------------- #
# Submit enrichment
# --------------------------------------------------------------------------- #
def test_enrich_candidates_adds_normalized_bounds_only_when_label_and_coords_agree():
explorer = _indexed_explorer()
settings = explorer.screen_index.exact_matches("Settings")[0]
explorer._register_label("X1", settings)
inside = {"label": "X1", "coords": SETTINGS_COORDS, "description": "gear"}
# 614px is 14px right of the element, inside the 2% (21.6px) tolerance.
near = {"label": "X1", "coords": [569, 200], "description": "gear"}
far = {"label": "X1", "coords": [100, 900], "description": "gear"}
unknown = {"label": "D7", "coords": SETTINGS_COORDS, "description": "gear"}
invalid = {"label": "X1", "coords": [5000, 5], "description": "gear"}
enriched = explorer._enrich_candidates([inside, near, far, unknown, invalid, "junk"])
assert enriched[0] == {**inside, "bounds": SETTINGS_BOUNDS, "source": "xml"}
assert enriched[1] == {**near, "bounds": SETTINGS_BOUNDS, "source": "xml"}
assert enriched[2:] == [far, unknown, invalid, "junk"]
assert "bounds" not in far and "bounds" not in inside # inputs are never mutated
def test_enrich_candidates_withholds_degenerate_bounds():
explorer = _indexed_explorer()
explorer._register_label("X9", ScreenElement("dot", (10, 10, 11, 11), "xml"))
cand = {"label": "X9", "coords": [9, 4]}
assert explorer._enrich_candidates([cand]) == [cand]
def test_native_submit_enriches_registered_candidates():
explorer = _indexed_explorer()
explorer._register_label("O2", explorer.screen_index.ocr_elements[0])
call = MagicMock()
call.name = "submit_answer"
call.args = {
"candidates": [
{"label": "O2", "coords": [185, 54], "description": "header"},
{"label": "D3", "coords": [900, 100], "description": "icon"},
],
"fallback_message": "",
}
turn_record = {"tool_calls": []}
raw = explorer._process_submit_answer([call], turn_record, [])
data = json.loads(raw)
assert list(data) == ["candidates", "fallback_message"]
assert data["candidates"][0] == {
"label": "O2",
"coords": [185, 54],
"description": "header",
"bounds": [92, 41, 277, 66],
"source": "ocr",
}
assert data["candidates"][1] == call.args["candidates"][1]
def test_universal_submit_enriches_registered_candidates():
explorer = _indexed_explorer()
explorer._register_label("1", explorer.screen_index.exact_matches("Settings")[0])
tool_calls = [
{
"name": "submit_answer",
"args": {
"candidates": [
{"label": "1", "coords": SETTINGS_COORDS, "description": "gear"},
{"label": "1", "coords": [5, 5000], "description": "bad"},
],
"fallback_message": "note",
},
"id": "c1",
}
]
data = json.loads(explorer._universal_submit_outcome(tool_calls))
assert data["fallback_message"] == "note"
assert data["candidates"] == [
{
"label": "1",
"coords": SETTINGS_COORDS,
"description": "gear",
"bounds": SETTINGS_BOUNDS,
"source": "xml",
}
]
# --------------------------------------------------------------------------- #
# Structural pre-pass
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize("version", ["flash", "pro"])
@pytest.mark.asyncio
async def test_prepass_answers_unique_exact_match_without_the_engine(screenshot, version):
explorer = Explorer(_context())
explorer._run_flash = AsyncMock()
explorer._run_loop = AsyncMock()
with patch(f"{FACADE}.StorageManager", return_value=_storage(None)):
raw = await explorer.run(" settings ", "", screenshot, _state(screenshot), version=version)
explorer._run_flash.assert_not_called()
explorer._run_loop.assert_not_called()
assert json.loads(raw) == {
"candidates": [
{
"label": "T1",
"coords": SETTINGS_COORDS,
"bounds": SETTINGS_BOUNDS,
"source": "xml",
"description": "Settings (exact text match)",
}
],
"fallback_message": "",
}
assert explorer.label_registry["T1"].text == "Settings"
@pytest.mark.asyncio
async def test_prepass_leaves_ambiguous_matches_to_the_engine(screenshot):
explorer = Explorer(_context())
engine = _outcome([{"label": "D1", "coords": [185, 854], "description": "Send"}])
explorer._run_flash = AsyncMock(return_value=engine)
with patch(f"{FACADE}.StorageManager", return_value=_storage(None)):
raw = await explorer.run("Send", "", screenshot, _state(screenshot), version="flash")
explorer._run_flash.assert_awaited_once_with("Send", screenshot)
assert raw == engine
@pytest.mark.asyncio
async def test_prepass_without_pipe_or_match_passes_the_query_through(screenshot):
explorer = Explorer(_context())
engine = _outcome(fallback="Failed to detect: gear icon")
explorer._run_flash = AsyncMock(return_value=engine)
with patch(f"{FACADE}.StorageManager", return_value=_storage(None)):
raw = await explorer.run("gear icon", "", screenshot, _state(screenshot), version="flash")
explorer._run_flash.assert_awaited_once_with("gear icon", screenshot)
assert raw == engine
@pytest.mark.asyncio
async def test_prepass_merges_partial_results_with_the_engine_outcome(screenshot):
explorer = Explorer(_context())
detected = {"label": "D1", "coords": [185, 854], "description": "Send"}
explorer._run_loop = AsyncMock(return_value=_outcome([detected], fallback="two Send buttons"))
with patch(f"{FACADE}.StorageManager", return_value=_storage(None)):
raw = await explorer.run(
"Settings | Send | Welcome", "", screenshot, _state(screenshot), version="pro"
)
# Only the unresolved part reaches the engine; "Welcome" is an OCR exact match.
assert explorer._run_loop.await_args.args[1] == "Send"
data = json.loads(raw)
assert [c["label"] for c in data["candidates"]] == ["T1", "T2", "D1"]
assert data["candidates"][1]["source"] == "ocr"
assert data["candidates"][1]["description"] == "Welcome (exact text match)"
assert data["candidates"][2] == detected
assert data["fallback_message"] == "two Send buttons"
def test_merge_prepass_keeps_structural_hits_over_an_unreadable_engine_answer():
prepass = [{"label": "T1", "coords": [1, 2]}]
merged = json.loads(Explorer._merge_prepass(prepass, "Error: max iterations"))
assert merged == {"candidates": prepass, "fallback_message": "Error: max iterations"}
assert Explorer._merge_prepass([], "raw") == "raw"
@pytest.mark.asyncio
async def test_prepass_never_runs_on_the_fly_ocr_for_flash(screenshot):
record = SimpleNamespace(ui_tree=HIERARCHY[:3], ocr_result=None)
explorer = Explorer(_context())
explorer._run_flash = AsyncMock(return_value=_outcome())
explorer._run_loop = AsyncMock(return_value=_outcome())
with (
patch(f"{FACADE}.StorageManager", return_value=_storage(record)),
patch(f"{FACADE}.is_ocr_configured", return_value=True),
patch(f"{FACADE}.perform_ocr", new_callable=AsyncMock, return_value=[]) as perform_ocr,
):
await explorer.run("gear icon", "", screenshot, _state(screenshot, []), version="flash")
perform_ocr.assert_not_awaited()
assert len(explorer.screen_index) == 3 # the record still feeds the index
await explorer.run("gear icon", "", screenshot, _state(screenshot, []), version="pro")
perform_ocr.assert_awaited_once()
tests/unit/agents/test_explorer_screen_index.py
+120
View File
@@ -0,0 +1,120 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for the Explorer's in-memory screen index."""
import pytest
from artemis.agents.explorer.screen_index import (
ScreenElement,
ScreenIndex,
normalize_text,
text_similarity,
)
W, H = 1080, 2400
def _node(text=None, bounds="[0,0][100,100]", desc=None, **attrs):
node = {"bounds": bounds, "class": "android.widget.TextView"}
if text is not None:
node["text"] = text
if desc is not None:
node["content-desc"] = desc
node.update(attrs)
return node
def _hierarchy():
return [
_node("Gmail", "[300,1600][500,1800]", clickable="true", **{"resource-id": "app:gmail"}),
_node(None, "[0,0][1080,2400]"), # unlabeled root wrapper is skipped
_node(desc="Settings", bounds="[900,50][1000,150]", clickable="true"),
{
"bounds": "[100,2100][900,2300]",
"class": "android.widget.FrameLayout [OCR]",
"ocr_elements": [{"text": "Search", "bounds": "[150,2150][350,2250]"}],
},
_node("Search", "[120,2120][380,2280]"), # overlaps the OCR "Search"
_node("Offscreen", "[-500,-500][-400,-400]"),
_node("Zero", "[10,10][10,10]"),
]
def test_normalize_and_similarity():
assert normalize_text(" Send\n Message ") == "send message"
assert text_similarity("gmail", "Gmail") == 1.0
assert text_similarity("send", "Send message") >= 0.85
assert text_similarity("ok", "Look here") < 0.85
assert text_similarity("", "x") == 0.0
def test_from_hierarchy_skips_invisible_and_unlabeled():
index = ScreenIndex.from_hierarchy(_hierarchy(), W, H)
texts = sorted(e.text for e in index.elements)
assert texts == ["Gmail", "Search", "Search", "Settings"]
assert len(index.ocr_elements) == 1
gmail = next(e for e in index.elements if e.text == "Gmail")
assert gmail.interactive and gmail.resource_id == "app:gmail" and gmail.source == "xml"
assert gmail.center == (400, 1700)
def test_from_hierarchy_tolerates_garbage():
index = ScreenIndex.from_hierarchy([None, "x", {"bounds": "bad"}, {}], W, H)
assert len(index) == 0
assert ScreenIndex.empty(W, H).search_text("x") == []
def test_search_text_orders_by_score_and_respects_threshold():
index = ScreenIndex.from_hierarchy(_hierarchy(), W, H)
matches = index.search_text("Gmai", threshold=0.6)
assert matches and matches[0].element.text == "Gmail"
assert all(m.score > 0.6 for m in matches)
assert index.search_text("zzzz", threshold=0.6) == []
assert len(index.search_text("Search", threshold=0.4, limit=1)) == 1
def test_exact_matches_collapse_overlapping_duplicates_preferring_xml():
index = ScreenIndex.from_hierarchy(_hierarchy(), W, H)
hits = index.exact_matches(" search ")
assert len(hits) == 1
assert hits[0].source == "xml"
assert index.exact_matches("settings")[0].text == "Settings"
assert index.exact_matches("") == []
assert index.exact_matches("nope") == []
def test_elements_at_reports_innermost_then_ocr():
elements = [
ScreenElement("outer", (0, 0, 1000, 1000), "xml"),
ScreenElement("middle", (100, 100, 900, 900), "xml", interactive=False),
ScreenElement("button", (400, 400, 600, 600), "xml", interactive=True),
ScreenElement("label", (450, 450, 550, 550), "ocr"),
]
index = ScreenIndex(elements, W, H)
found = index.elements_at(500, 500)
# innermost interactive node stops the UI-tree walk; OCR appended after
assert [e.text for e in found] == ["button", "label"]
found = index.elements_at(150, 150)
assert [e.text for e in found] == ["middle", "outer"]
assert index.elements_at(1050, 1050) == []
def test_describe_and_overlap():
a = ScreenElement("A", (0, 0, 100, 100), "xml", class_name="Btn", resource_id="id/a")
b = ScreenElement("B", (50, 50, 150, 150), "ocr")
assert "class=Btn" in a.describe() and "id=id/a" in a.describe()
assert "source=ocr" in b.describe()
assert a.overlap_ratio(b) == pytest.approx(0.25)
assert a.overlap_ratio(ScreenElement("C", (200, 200, 300, 300), "xml")) == 0.0
tests/unit/agents/test_explorer_tiers.py
+555
View File
@@ -0,0 +1,555 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Explorer tier integration: engine selection, tool exposure, caching, prompts."""
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
from google.genai import types
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
import pytest
from artemis.agents.explorer.explorer import Explorer
from artemis.agents.explorer.tiers import EXPLORER_TIERS, PERCEPTION_TOOLS, get_tier
from artemis.config import settings
from artemis.context import ArtemisContext
from artemis.graph.state import State
FACADE = "artemis.agents.explorer.explorer"
def _context(model: str = "gemini-3.8-flash", denylist: list[str] | None = None) -> MagicMock:
ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = None
ctx.llm_config = MagicMock()
ctx.llm_config.explorer = MagicMock()
ctx.llm_config.explorer.model = model
ctx.llm_config.explorer.temperature = 0.1
ctx.llm_config.explorer.thinking_level = None
ctx.llm_config.explorer.fallback = None
ctx.agent_config = MagicMock()
ctx.agent_config.denylisted_tools = {"explorer": denylist or []}
ctx.agent_config.explorer = SimpleNamespace(caching=None)
ctx.execution_setup = SimpleNamespace(explorer=SimpleNamespace(caching=None))
return ctx
def _state(screenshot: str) -> MagicMock:
state = MagicMock(spec=State)
state.latest_screenshot = screenshot
state.latest_ui_hierarchy = []
state.operator_raw_data = {}
return state
def _submit_response(coords=(500, 600)) -> MagicMock:
call = MagicMock()
call.name = "submit_answer"
call.args = {
"candidates": [{"label": "D1", "coords": list(coords), "description": "target"}],
"fallback_message": "",
}
response = MagicMock()
response.function_calls = [call]
response.candidates = [MagicMock()]
response.candidates[0].content = types.Content(
role="model",
parts=[types.Part.from_function_call(name="submit_answer", args=call.args)],
)
return response
def _native_client(responses) -> MagicMock:
client = MagicMock()
client.aio.models.generate_content = AsyncMock(side_effect=list(responses))
cache = SimpleNamespace(name="cachedContents/abc")
client.aio.caches.create = AsyncMock(return_value=cache)
client.aio.caches.delete = AsyncMock()
client.aio.files.upload = AsyncMock(
return_value=SimpleNamespace(uri="files/1", mime_type="image/jpeg", name="files/1")
)
client.aio.files.delete = AsyncMock()
return client
@pytest.fixture
def screenshot(tmp_path):
path = tmp_path / "screen.jpg"
path.write_bytes(b"fake-jpeg-bytes")
return str(path)
@pytest.fixture(autouse=True)
def _isolated_env(monkeypatch):
monkeypatch.delenv("ARTEMIS_EXPLORER_CACHING", raising=False)
monkeypatch.setenv("ARTEMIS_USE_FILE_API", "false")
# --------------------------------------------------------------------------- #
# Tier -> turn budget / tool exposure
# --------------------------------------------------------------------------- #
def test_construction_creates_no_genai_client():
with patch(f"{FACADE}.genai.Client") as client_cls:
explorer = Explorer(_context())
client_cls.assert_not_called()
assert explorer.client is None
assert explorer.tier is None
@pytest.mark.parametrize(
("version", "max_turns", "expected_tools"),
[
("pro", 3, {"ask_perception_tool", "submit_answer"}),
("ultra", 8, PERCEPTION_TOOLS | {"submit_answer"}),
],
)
def test_tier_sets_turn_budget_and_exposed_tools(version, max_turns, expected_tools):
explorer = Explorer(_context())
with patch(f"{FACADE}.is_ocr_configured", return_value=True):
explorer._apply_tier(get_tier(version))
native_names = {t.name for t in explorer.get_exposed_tools()}
universal_names = {t["function"]["name"] for t in explorer._universal_exposed_tools()}
assert explorer.max_turns == max_turns
assert native_names == expected_tools
assert universal_names == expected_tools
assert explorer.denylisted_tools >= get_tier(version).hidden_tools
def test_unconfigured_ocr_and_user_denylist_hide_tools_on_ultra():
explorer = Explorer(_context(denylist=["inspect_region"]))
with patch(f"{FACADE}.is_ocr_configured", return_value=False):
explorer._apply_tier(get_tier("ultra"))
names = {t.name for t in explorer.get_exposed_tools()}
assert "get_ocr_list" not in names
assert "inspect_region" not in names
assert {"detect_objects", "ask_image_processor", "ask_perception_tool"} <= names
# --------------------------------------------------------------------------- #
# Engine selection
# --------------------------------------------------------------------------- #
def test_gemini_model_without_google_key_uses_universal_engine():
with patch.object(settings, "GOOGLE_API_KEY", None):
explorer = Explorer(_context(model="gemini-3.8-flash"))
assert explorer.use_native_gemini is False
def test_gemini_model_with_shared_client_uses_native_engine():
ctx = _context(model="google/gemini-3.8-pro")
ctx._genai_client = MagicMock()
with patch.object(settings, "GOOGLE_API_KEY", None):
explorer = Explorer(ctx)
assert explorer.use_native_gemini is True
assert explorer.model_name == "gemini-3.8-pro"
@pytest.mark.asyncio
async def test_universal_run_without_google_key_never_touches_genai(screenshot):
ctx = _context(model="claude-sonnet-4")
response = AIMessage(
content="",
tool_calls=[
{
"name": "submit_answer",
"args": {
"candidates": [{"label": "1", "coords": [100, 200], "description": "ok"}],
"fallback_message": "",
},
"id": "call_1",
}
],
)
llm = MagicMock()
bound = MagicMock()
bound.ainvoke = AsyncMock(return_value=response)
llm.bind_tools.return_value = bound
storage = MagicMock()
storage.get_image.return_value = None
with (
patch.object(settings, "GOOGLE_API_KEY", None),
patch(f"{FACADE}.genai.Client") as client_cls,
patch(f"{FACADE}.get_llm", return_value=llm),
patch(f"{FACADE}.StorageManager", return_value=storage),
):
explorer = Explorer(ctx)
raw = await explorer.run("find it", "", screenshot, _state(screenshot), version="pro")
client_cls.assert_not_called()
outcome = json.loads(raw)
assert outcome["candidates"][0]["coords"] == [100, 200]
# The universal engine only sees the pro tier's tools.
bound_names = {t["function"]["name"] for t in llm.bind_tools.call_args_list[0].args[0]}
assert bound_names == {"ask_perception_tool", "submit_answer"}
# --------------------------------------------------------------------------- #
# Universal dispatch delivers tool images
# --------------------------------------------------------------------------- #
@pytest.mark.asyncio
async def test_universal_dispatch_appends_image_message(tmp_path):
explorer = Explorer(_context(model="claude-sonnet-4"))
img_a = tmp_path / "a.jpg"
img_b = tmp_path / "b.jpg"
img_a.write_bytes(b"aaa")
img_b.write_bytes(b"bbb")
explorer.exec_ask_perception_tool = AsyncMock(
return_value={"text": "perception text", "image_paths": [str(img_a), str(tmp_path / "x")]}
)
explorer.exec_inspect_region = AsyncMock(
return_value={"text": "zoomed", "image_path": str(img_b)}
)
messages = []
tool_calls = [
{
"name": "ask_perception_tool",
"args": {"search_query": "q", "nx": 1, "ny": 2, "detect_queries": ["q"]},
"id": "c1",
},
{
"name": "inspect_region",
"args": {"x_min": 0, "y_min": 0, "x_max": 10, "y_max": 10},
"id": "c2",
},
]
await explorer._universal_dispatch_tools(tool_calls, messages, turn=1)
assert [type(m) for m in messages] == [ToolMessage, ToolMessage, HumanMessage]
assert messages[0].content == "perception text"
human = messages[2]
assert human.content[0] == {
"type": "text",
"text": "[Annotated image(s) returned by: ask_perception_tool, inspect_region]",
}
image_blocks = human.content[1:]
assert len(image_blocks) == 2 # the missing file is skipped
assert all(b["type"] == "image_url" for b in image_blocks)
assert image_blocks[0]["image_url"]["url"].startswith("data:image/jpeg;base64,")
@pytest.mark.asyncio
async def test_universal_dispatch_without_images_adds_no_human_message():
explorer = Explorer(_context(model="claude-sonnet-4"))
explorer.exec_get_ocr_list = AsyncMock(return_value={"text": "none", "image_path": None})
messages = []
await explorer._universal_dispatch_tools(
[{"name": "get_ocr_list", "args": {}, "id": "c1"}], messages, turn=1
)
assert [type(m) for m in messages] == [ToolMessage]
@pytest.mark.asyncio
async def test_universal_dispatch_rejects_denylisted_tool():
explorer = Explorer(_context(model="claude-sonnet-4"))
explorer._apply_tier(get_tier("pro"))
explorer.exec_inspect_region = AsyncMock()
messages = []
await explorer._universal_dispatch_tools(
[{"name": "inspect_region", "args": {}, "id": "c1"}], messages, turn=1
)
explorer.exec_inspect_region.assert_not_called()
assert "denylisted" in messages[0].content
# --------------------------------------------------------------------------- #
# Caching precedence
# --------------------------------------------------------------------------- #
def test_caching_tier_defaults():
explorer = Explorer(_context())
with patch.object(settings, "EXPLORER_CACHING", None):
assert explorer._resolve_caching(get_tier("pro"), None) is False
assert explorer._resolve_caching(get_tier("ultra"), None) is True
def test_caching_precedence(monkeypatch):
ctx = _context()
explorer = Explorer(ctx)
pro = get_tier("pro")
with patch.object(settings, "EXPLORER_CACHING", None):
ctx.agent_config.explorer = SimpleNamespace(caching=True)
assert explorer._resolve_caching(pro, None) is True # agent config beats tier
monkeypatch.setenv("ARTEMIS_EXPLORER_CACHING", "false")
assert explorer._resolve_caching(pro, None) is False # env beats agent config
assert explorer._resolve_caching(pro, True) is True # explicit beats env
with patch.object(settings, "EXPLORER_CACHING", True):
monkeypatch.delenv("ARTEMIS_EXPLORER_CACHING")
ctx.agent_config.explorer = SimpleNamespace(caching=None)
ctx.execution_setup = SimpleNamespace(explorer=SimpleNamespace(caching=False))
assert explorer._resolve_caching(pro, None) is False # execution setup beats settings
ctx.execution_setup = SimpleNamespace(explorer=SimpleNamespace(caching=None))
assert explorer._resolve_caching(pro, None) is True # settings beat tier
def test_caching_ignores_mock_config_values():
"""A MagicMock context must fall through to the tier default, not force True."""
ctx = MagicMock()
explorer = Explorer(ctx)
with patch.object(settings, "EXPLORER_CACHING", None):
assert explorer._resolve_caching(get_tier("pro"), None) is False
# --------------------------------------------------------------------------- #
# Tier-aware prompt
# --------------------------------------------------------------------------- #
def test_pro_prompt_only_describes_exposed_tools():
explorer = Explorer(_context())
explorer._apply_tier(get_tier("pro"))
prompt, error = explorer._build_prompt_template(get_tier("pro"))
assert error is None
assert "ask_perception_tool" in prompt
assert "inspect_region" not in prompt
assert "Image Pool" not in prompt
assert "ask_image_processor" not in prompt
assert "get_ocr_list" not in prompt
assert "# TOOL DENYLIST" not in prompt
assert "# EXECUTION CONSTRAINT" in prompt
assert "maximum of 3 turns" in prompt
def test_ultra_prompt_describes_all_tools():
explorer = Explorer(_context())
with patch(f"{FACADE}.is_ocr_configured", return_value=True):
explorer._apply_tier(get_tier("ultra"))
prompt, error = explorer._build_prompt_template(get_tier("ultra"))
assert error is None
for tool in PERCEPTION_TOOLS:
assert tool in prompt
assert "Image Pool" in prompt
assert "maximum of 8 turns" in prompt
assert "# TOOL DENYLIST" not in prompt
def test_prompt_denylist_section_only_for_user_denied_tier_tools():
explorer = Explorer(_context(denylist=["inspect_region", "search_xml_ocr"]))
with patch(f"{FACADE}.is_ocr_configured", return_value=True):
explorer._apply_tier(get_tier("ultra"))
ultra_prompt, _ = explorer._build_prompt_template(get_tier("ultra"))
explorer._apply_tier(get_tier("pro"))
pro_prompt, _ = explorer._build_prompt_template(get_tier("pro"))
assert "# TOOL DENYLIST" in ultra_prompt
assert "cannot be used: inspect_region" in ultra_prompt
assert "search_xml_ocr" not in ultra_prompt
# Denied tools lose their guidance bullets too.
assert "Targeted Verification" not in ultra_prompt
# On pro, inspect_region is tier-hidden, so it is never mentioned.
assert "# TOOL DENYLIST" not in pro_prompt
assert "inspect_region" not in pro_prompt
def test_prompt_sections_render_shape():
explorer = Explorer(_context())
explorer._apply_tier(get_tier("pro"))
prompt, _ = explorer._build_prompt_template(get_tier("pro"))
lines = prompt.splitlines()
assert lines[0] == "# IDENTITY"
assert any(line.startswith("# OPERATING PRINCIPLES") for line in lines)
body = [line for line in lines if line and not line.startswith("#")]
assert all(line.startswith("- ") or line.startswith("You are") for line in body)
# --------------------------------------------------------------------------- #
# Flash path
# --------------------------------------------------------------------------- #
@pytest.mark.asyncio
async def test_flash_run_returns_detected_candidates(screenshot):
detection = AsyncMock(
return_value={
"detected": [
{"label": "gear icon", "point": [900, 100]},
{"label": "send button", "point": [500, 950]},
],
"failed": [],
}
)
with (
patch(f"{FACADE}._run_object_detection", detection),
patch(f"{FACADE}.genai.Client") as client_cls,
):
explorer = Explorer(_context())
raw = await explorer.run(
"gear icon | send button", "", screenshot, _state(screenshot), version="flash"
)
client_cls.assert_not_called()
outcome = json.loads(raw)
assert [c["label"] for c in outcome["candidates"]] == ["D1", "D2"]
assert outcome["candidates"][0] == {
"label": "D1",
"coords": [900, 100],
"description": "gear icon",
}
assert outcome["fallback_message"] == ""
assert detection.call_args.args[2] == ["gear icon", "send button"]
@pytest.mark.asyncio
async def test_flash_run_reports_missing_detection(screenshot):
with patch(f"{FACADE}._run_object_detection", AsyncMock(return_value={"detected": []})):
raw = await Explorer(_context()).run(
"ghost", "", screenshot, _state(screenshot), version="flash"
)
assert json.loads(raw) == {"candidates": [], "fallback_message": "Failed to detect: ghost"}
# --------------------------------------------------------------------------- #
# Native engine: async SDK usage and resource lifecycle
# --------------------------------------------------------------------------- #
@pytest.mark.asyncio
async def test_native_ultra_run_uses_async_cache_api(screenshot):
ctx = _context()
client = _native_client([_submit_response()])
ctx._genai_client = client
storage = MagicMock()
storage.get_image.return_value = None
with (
patch(f"{FACADE}.genai.Client") as client_cls,
patch(f"{FACADE}.StorageManager", return_value=storage),
patch.object(settings, "EXPLORER_CACHING", None),
):
explorer = Explorer(ctx)
raw = await explorer.run("target", "", screenshot, _state(screenshot), version="ultra")
client_cls.assert_not_called()
assert json.loads(raw)["candidates"][0]["coords"] == [500, 600]
client.aio.caches.create.assert_awaited_once()
client.aio.caches.delete.assert_awaited_once_with(name="cachedContents/abc")
config = client.aio.models.generate_content.call_args.kwargs["config"]
assert config.cached_content == "cachedContents/abc"
@pytest.mark.asyncio
async def test_native_pro_run_skips_cache_by_default(screenshot):
ctx = _context()
client = _native_client([_submit_response()])
ctx._genai_client = client
storage = MagicMock()
storage.get_image.return_value = None
with (
patch(f"{FACADE}.StorageManager", return_value=storage),
patch.object(settings, "EXPLORER_CACHING", None),
):
await Explorer(ctx).run("target", "", screenshot, _state(screenshot), version="pro")
client.aio.caches.create.assert_not_awaited()
client.aio.caches.delete.assert_not_awaited()
config = client.aio.models.generate_content.call_args.kwargs["config"]
assert {t.name for t in config.tools[0].function_declarations} == {
"ask_perception_tool",
"submit_answer",
}
@pytest.mark.asyncio
async def test_native_file_api_uploads_and_deletes_asynchronously(screenshot, monkeypatch):
monkeypatch.setenv("ARTEMIS_USE_FILE_API", "true")
ctx = _context()
client = _native_client([_submit_response()])
ctx._genai_client = client
storage = MagicMock()
storage.get_image.return_value = None
with patch(f"{FACADE}.StorageManager", return_value=storage):
await Explorer(ctx).run("target", "", screenshot, _state(screenshot), version="pro")
client.aio.files.upload.assert_awaited_once_with(file=screenshot)
client.aio.files.delete.assert_awaited_once_with(name="files/1")
@pytest.mark.asyncio
async def test_run_closes_http_client_on_both_engines(screenshot):
storage = MagicMock()
storage.get_image.return_value = None
# Native engine
ctx = _context()
client = _native_client([_submit_response()])
ctx._genai_client = client
native = Explorer(ctx)
native.http_client = SimpleNamespace(aclose=AsyncMock())
with patch(f"{FACADE}.StorageManager", return_value=storage):
await native.run("t", "", screenshot, _state(screenshot), version="pro")
native_close = native.http_client # already reset
assert native_close is None
# Universal engine (model call fails -> finally still closes)
llm = MagicMock()
llm.bind_tools.return_value.ainvoke = AsyncMock(side_effect=RuntimeError("boom"))
universal = Explorer(_context(model="claude-sonnet-4"))
http_client = SimpleNamespace(aclose=AsyncMock())
universal.http_client = http_client
with (
patch(f"{FACADE}.StorageManager", return_value=storage),
patch(f"{FACADE}.get_llm", return_value=llm),
pytest.raises(RuntimeError),
):
await universal.run("t", "", screenshot, _state(screenshot), version="pro")
http_client.aclose.assert_awaited_once()
assert universal.http_client is None
def test_missing_tool_calls_without_candidates_does_not_crash():
explorer = Explorer(_context())
response = MagicMock()
response.candidates = None
contents = []
turn_record = {"tool_calls": []}
explorer._handle_missing_tool_calls(contents, response, turn_record, ["hi"], 3, 1)
assert contents[0].role == "model"
assert contents[0].parts[0].text == "hi"
assert "2 more" in contents[1].parts[0].text
assert turn_record["tool_calls"][0]["name"] == "hallucinated_plain_text"
def test_tier_table_is_single_source_for_versions():
assert set(EXPLORER_TIERS) == {"flash", "pro", "ultra"}
assert get_tier("bogus").name == "flash"
@pytest.mark.parametrize("version", [None, "", "turbo", "PRO ", "Ultra"])
def test_get_tier_normalizes_or_falls_back(version):
"""Unknown or empty names fall back to the default tier; case and spaces are tolerated."""
tier = get_tier(version)
normalized = str(version or "").strip().lower()
expected = normalized if normalized in EXPLORER_TIERS else "flash"
assert tier.name == expected
tests/unit/agents/test_explorer_universal.py
+1 -2
View File
@@ -116,9 +116,8 @@ async def test_run_universal_explorer_success(mock_context, tmp_path):
screenshot_path=str(fake_screenshot),
state=mock_state,
minimal_list="",
version="pro",
prompt_template="You are an explorer.",
max_iterations=3,
max_turns=3,
)
res = json.loads(res_str)
tests/unit/agents/test_failure_analyzer.py
-213
View File
@@ -1,213 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Universal Failure Analyzer."""
import base64
from unittest.mock import AsyncMock, MagicMock, Mock, patch
import pytest
from langchain_core.messages import AIMessage
from artemis.agents.validator.failure_analyzer import (
FailureAnalyzer,
PixelTargetDisappearedStrategy,
TargetDisappearedStrategy,
ValidationErrorCategory,
VALIDATOR_TOOLS_DECLARATION,
)
from artemis.context import ArtemisContext
class DummyState:
def __init__(self, latest_screenshot=None):
self.latest_screenshot = latest_screenshot
self.indexed_points = []
self.indexed_elements = []
@pytest.fixture
def mock_context(tmp_path):
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = Mock()
mock_llm_cfg = Mock()
mock_llm_cfg.model = "gemini-2.5-pro"
mock_llm_cfg.temperature = 0.1
ctx.llm_config.get_agent.return_value = mock_llm_cfg
ctx.device = Mock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.data_engine = Mock()
ctx.data_engine.base_dir = tmp_path
ctx.data_engine.get_agent_friendly_steps.return_value = []
ctx.adb_client = None
ctx.driver = Mock()
return ctx
@pytest.mark.asyncio
async def test_failure_analyzer_cannot_fix(mock_context, tmp_path):
"""Test that FailureAnalyzer natively executes and handles 'cannot_fix' report."""
screenshot_file = tmp_path / "shot.jpg"
screenshot_file.write_bytes(b"dummy_bytes")
state = DummyState(latest_screenshot=str(screenshot_file))
failed_action = {"action": "click", "coordinates": [100, 200]}
error_msg = "Element not found"
analyzer = FailureAnalyzer(mock_context)
mock_llm = MagicMock()
mock_response = AIMessage(
content="I see the error.",
tool_calls=[
{
"name": "report_failure_analysis",
"args": {
"status": "cannot_fix",
"analysis": "The element was definitely not there.",
},
"id": "call_1",
}
],
)
mock_bound = MagicMock()
mock_bound.ainvoke = AsyncMock(return_value=mock_response)
mock_llm.bind_tools.return_value = mock_bound
with (
patch("artemis.agents.validator.failure_analyzer.get_llm", return_value=mock_llm),
patch(
"artemis.agents.validator.failure_analyzer.UnifiedMobileController"
) as mock_controller_cls,
):
mock_controller = Mock()
mock_controller_cls.return_value = mock_controller
mock_device_data = Mock()
mock_device_data.base64 = base64.b64encode(b"mocked_screenshot").decode()
mock_device_data.elements = []
mock_device_data.width = 1080
mock_device_data.height = 2400
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
mock_controller.get_ui_elements = AsyncMock(return_value=mock_device_data.elements)
result = await analyzer.analyze(
state,
failed_action,
error_msg,
pre_screenshot=base64.b64encode(b"dummy").decode(),
post_screenshot=base64.b64encode(b"dummy").decode(),
)
assert result["status"] == "cannot_fix"
assert "not there" in result["analysis"]
assert result["new_remaining_actions"] == []
def test_report_failure_analysis_tool_declaration():
"""Test that report_failure_analysis tool declaration is correct and matches."""
report_tool = next(
(t for t in VALIDATOR_TOOLS_DECLARATION if t.name == "report_failure_analysis"),
None,
)
assert report_tool is not None
props = report_tool.parameters.get("properties", {})
assert "analysis" in props
assert "single paragraph" in props["analysis"]["description"]
@pytest.mark.asyncio
async def test_failure_analyzer_strategy_routing(mock_context):
"""Test that FailureAnalyzer selects appropriate strategies based on error category."""
analyzer = FailureAnalyzer(mock_context)
strat_disappeared = analyzer._select_strategy(ValidationErrorCategory.TARGET_DISAPPEARED)
assert isinstance(strat_disappeared, TargetDisappearedStrategy)
assert "target_disappeared" in strat_disappeared.get_prompt_template_name()
strat_pixel = analyzer._select_strategy(ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED)
assert isinstance(strat_pixel, PixelTargetDisappearedStrategy)
assert "pixel_target_disappeared" in strat_pixel.get_prompt_template_name()
@pytest.mark.asyncio
async def test_failure_analyzer_click_sequence(mock_context, tmp_path):
"""Test click_sequence execution in FailureAnalyzer."""
screenshot_file = tmp_path / "shot.jpg"
screenshot_file.write_bytes(b"dummy_bytes")
state = DummyState(latest_screenshot=str(screenshot_file))
analyzer = FailureAnalyzer(mock_context)
# Turn 1: Call click_sequence tool with direct coordinates
call_step_1 = AIMessage(
content="Let me click in sequence.",
tool_calls=[
{
"name": "click_sequence",
"args": {"sequence": [[100, 200], [300, 400]]},
"id": "call_seq_1",
}
],
)
# Turn 2: Report failure fixed
call_step_2 = AIMessage(
content="Repaired successfully.",
tool_calls=[
{
"name": "report_failure_analysis",
"args": {
"status": "fixed",
"analysis": "Sequence clicked successfully.",
},
"id": "call_report_2",
}
],
)
mock_llm = MagicMock()
mock_bound = MagicMock()
mock_bound.ainvoke = AsyncMock(side_effect=[call_step_1, call_step_2])
mock_llm.bind_tools.return_value = mock_bound
with (
patch("artemis.agents.validator.failure_analyzer.get_llm", return_value=mock_llm),
patch(
"artemis.agents.validator.failure_analyzer.UnifiedMobileController"
) as mock_controller_cls,
):
mock_controller = Mock()
mock_controller_cls.return_value = mock_controller
mock_device_data = Mock()
mock_device_data.base64 = base64.b64encode(b"mocked_screenshot").decode()
mock_device_data.elements = []
mock_device_data.width = 1080
mock_device_data.height = 2400
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
mock_controller.get_ui_elements = AsyncMock(return_value=mock_device_data.elements)
mock_controller.tap_at = AsyncMock(return_value=Mock(error=None))
result = await analyzer.analyze(
state,
{"action": "click", "coordinates": [100, 200]},
"Element not found",
pre_screenshot=base64.b64encode(b"dummy").decode(),
post_screenshot=base64.b64encode(b"dummy").decode(),
error_category=ValidationErrorCategory.TARGET_DISAPPEARED,
)
assert result["status"] == "fixed"
assert mock_controller.tap_at.call_count == 2
tests/unit/agents/test_failure_analyzer_tools.py
-466
View File
@@ -1,466 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
from unittest.mock import AsyncMock, Mock, patch
from artemis.agents.validator.failure_analyzer import (
_exec_click,
_exec_click_sequence,
_exec_input_text,
_exec_long_press,
_exec_manage_app,
_exec_press_key,
_exec_swipe,
_exec_wait_for_delay,
)
from artemis.context import ArtemisContext
import pytest
class DummyState:
def __init__(self, latest_screenshot=None):
self.latest_screenshot = latest_screenshot
self.indexed_points = []
self.indexed_elements = []
@pytest.fixture
def mock_context(tmp_path):
ctx = Mock(spec=ArtemisContext)
ctx.device = Mock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.data_engine = Mock()
ctx.data_engine.base_dir = tmp_path
ctx.data_engine.get_or_create_image.return_value = "dummy_image_name"
ctx.data_engine.get_image_path.return_value = tmp_path / "dummy_image_name.jpg"
return ctx
@pytest.fixture
def mock_controller():
controller = Mock()
mock_device_data = Mock()
mock_device_data.base64 = base64.b64encode(b"mocked_screenshot").decode()
mock_device_data.elements = []
mock_device_data.width = 1080
mock_device_data.height = 2400
controller.get_screen_data = AsyncMock(return_value=mock_device_data)
controller.get_ui_elements = AsyncMock(return_value=mock_device_data.elements)
controller.tap_at = AsyncMock(return_value=Mock(error=None))
controller.type_text = AsyncMock(return_value=True)
controller.erase_text = AsyncMock(return_value=True)
controller.swipe_coords = AsyncMock(return_value=None)
controller.press_enter = AsyncMock(return_value=True)
controller.go_back = AsyncMock(return_value=True)
controller.go_home = AsyncMock(return_value=True)
controller.press_key = AsyncMock(return_value=True)
controller.terminate_app = AsyncMock(return_value=True)
return controller
# ==========================================
# SLIDE 1: click
# ==========================================
@pytest.mark.asyncio
async def test_click_happy_path(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_click(
target=[500, 600],
state=state,
controller=mock_controller,
ctx=mock_context,
times=2,
delay_ms=150,
)
assert "Clicked at [500, 600]" in outcome
assert mock_controller.tap_at.call_count == 1
mock_controller.tap_at.assert_any_call(540, 1440, times=2, delay_ms=150)
@pytest.mark.asyncio
async def test_click_input_validation(mock_context, mock_controller):
state = DummyState()
# Coordinates holding non-integer values
outcome, _, _, _ = await _exec_click(
target=["invalid", "invalid"],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error during click:" in outcome
@pytest.mark.asyncio
async def test_click_edge_cases(mock_context, mock_controller):
state = DummyState()
# Underflow/overflow coordinates should be clamped to valid boundary
outcome, _, _, _ = await _exec_click(
target=[-10, 1050],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Clicked at [-10, 1050]" in outcome
mock_controller.tap_at.assert_called_with(0, 2399, times=1, delay_ms=100)
@pytest.mark.asyncio
async def test_click_graceful_failure(mock_context, mock_controller):
state = DummyState()
mock_controller.tap_at.return_value = Mock(error="Click failed connection lost")
outcome, _, _, _ = await _exec_click(
target=[500, 600],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error executing click: Click failed connection lost" in outcome
# ==========================================
# SLIDE 2: click_sequence
# ==========================================
@pytest.mark.asyncio
async def test_click_sequence_happy_path(mock_context, mock_controller):
state = DummyState()
state.indexed_points = [[540, 1200]] # Mapped from index 1 (1-indexed)
outcome, img_bytes, shot_path, xml_list = await _exec_click_sequence(
sequence=[1, [300, 400]],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
assert mock_controller.tap_at.call_count == 2
mock_controller.tap_at.assert_any_call(540, 1200)
mock_controller.tap_at.assert_any_call(324, 960)
@pytest.mark.asyncio
async def test_click_sequence_input_validation(mock_context, mock_controller):
state = DummyState()
state.indexed_points = [[540, 1200], [270, 600]]
# Subsequent elements can also be integers (mixed array of indexes and coordinates)
outcome, _, _, _ = await _exec_click_sequence(
sequence=[1, 2, [300, 400]],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
assert mock_controller.tap_at.call_count == 3
mock_controller.tap_at.assert_any_call(540, 1200)
mock_controller.tap_at.assert_any_call(270, 600)
mock_controller.tap_at.assert_any_call(324, 960)
# Reset indexed_points since _exec_click_sequence overwrites them
# via _capture_screenshot_and_parse_ui
state.indexed_points = [[540, 1200], [270, 600]]
# Index out of bounds
outcome, _, _, _ = await _exec_click_sequence(
sequence=[5, [300, 400]],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error during click sequence: Index 5 is out of range" in outcome
@pytest.mark.asyncio
async def test_click_sequence_edge_cases(mock_context, mock_controller):
state = DummyState()
# Coordinates in sequence outside range should be clamped
outcome, _, _, _ = await _exec_click_sequence(
sequence=[[0, 0], [1200, -50]],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
mock_controller.tap_at.assert_any_call(0, 0)
mock_controller.tap_at.assert_any_call(1079, 0)
@pytest.mark.asyncio
async def test_click_sequence_graceful_failure(mock_context, mock_controller):
state = DummyState()
mock_controller.tap_at.side_effect = [
Mock(error=None),
Mock(error="Connection error on step 2"),
]
outcome, _, _, _ = await _exec_click_sequence(
sequence=[[100, 200], [300, 400]],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error executing click at" in outcome
assert "Connection error on step 2" in outcome
@pytest.mark.asyncio
async def test_click_sequence_stringified_formats(mock_context, mock_controller):
state = DummyState()
state.indexed_points = [[540, 1200]]
# Test stringified coordinate formats commonly generated by LLMs:
# 1. string representation of lists: ["[500, 280]", "[884, 362]"]
outcome, _, _, _ = await _exec_click_sequence(
sequence=["[500, 280]", "[884, 362]"],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
mock_controller.tap_at.assert_any_call(540, 672)
mock_controller.tap_at.assert_any_call(954, 868)
# 2. comma-separated strings: ["500,280", "884,362"]
outcome, _, _, _ = await _exec_click_sequence(
sequence=["500,280", "884,362"],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
# 3. whole sequence as a string
outcome, _, _, _ = await _exec_click_sequence(
sequence="[[500, 280], [884, 362]]",
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
# ==========================================
# SLIDE 3: long_press
# ==========================================
@pytest.mark.asyncio
async def test_long_press_happy_path(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_long_press(
target=[500, 500],
duration=1500,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Long pressed at [500, 500]" in outcome
mock_controller.tap_at.assert_called_once_with(
540, 1200, long_press=True, long_press_duration=1500
)
@pytest.mark.asyncio
async def test_long_press_input_validation(mock_context, mock_controller):
state = DummyState()
# Invalid target type
outcome, _, _, _ = await _exec_long_press(
target="not-a-list",
duration=1500,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error during long press:" in outcome
# ==========================================
# SLIDE 4: input_text
# ==========================================
@pytest.mark.asyncio
async def test_input_text_happy_path(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_input_text(
text="hello",
target=[500, 500],
clear_exist=True,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Executed typing 'hello'." in outcome
# Focus goes through ensure_focus_at_coords (keyword-style tap), and typing
# never re-clears: the clear already happened via erase_text.
mock_controller.tap_at.assert_called_once_with(x=540, y=1200)
mock_controller.erase_text.assert_called_once()
mock_controller.type_text.assert_called_once_with("hello", clear_existing=False)
@pytest.mark.asyncio
async def test_input_text_input_validation(mock_context, mock_controller):
state = DummyState()
outcome, _, _, _ = await _exec_input_text(
text="hello",
target="invalid",
clear_exist=True,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error during input text:" in outcome
# ==========================================
# SLIDE 5: swipe
# ==========================================
@pytest.mark.asyncio
async def test_swipe_happy_path_direction(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_swipe(
action="up",
duration=400,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Swipe completed successfully" in outcome
mock_controller.swipe_coords.assert_called_once_with(648, 1680, 648, 720, 400)
@pytest.mark.asyncio
async def test_swipe_happy_path_coords(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_swipe(
action=[100, 900, 100, 100],
duration=400,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Swipe completed successfully" in outcome
mock_controller.swipe_coords.assert_called_once_with(108, 2160, 108, 240, 400)
@pytest.mark.asyncio
async def test_swipe_input_validation(mock_context, mock_controller):
state = DummyState()
outcome, _, _, _ = await _exec_swipe(
action="invalid_direction",
duration=400,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error during swipe: Invalid direction:" in outcome
# ==========================================
# SLIDE 6: press_key
# ==========================================
@pytest.mark.asyncio
async def test_press_key_happy_path(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_press_key(
key="BACK",
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Executed key press 'BACK'." in outcome
mock_controller.go_back.assert_called_once()
@pytest.mark.asyncio
async def test_press_key_input_validation(mock_context, mock_controller):
state = DummyState()
outcome, _, _, _ = await _exec_press_key(
key="INVALID_KEY",
state=state,
controller=mock_controller,
ctx=mock_context,
)
# Unknown keycodes are forwarded verbatim to the driver (historical adb_server
# behavior); the mock driver accepts them, so the press reports success.
assert "Executed key press 'INVALID_KEY'." in outcome
mock_controller.press_key.assert_called_once_with("INVALID_KEY")
# ==========================================
# SLIDE 7: manage_app
# ==========================================
@pytest.mark.asyncio
async def test_manage_app_happy_path(mock_context, mock_controller):
state = DummyState()
with (
# The device-call bodies moved into the actuator layer, which imports these
# lazily from their source modules -- patch them at the source.
patch("artemis.tools.mobile.launch_app.find_package") as mock_find_package,
patch("artemis.utils.app_launch_utils.launch_app_with_retries") as mock_launch,
):
mock_find_package.return_value = "com.google.android.youtube"
mock_launch.return_value = (True, "")
outcome, img_bytes, shot_path, xml_list = await _exec_manage_app(
action="launch",
app_name="YouTube",
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Launched app 'YouTube' (com.google.android.youtube) successfully." in outcome
mock_find_package.assert_called_once_with(mock_context, "YouTube", use_fallback=False)
mock_launch.assert_called_once_with(mock_context, "com.google.android.youtube")
@pytest.mark.asyncio
async def test_manage_app_input_validation(mock_context, mock_controller):
state = DummyState()
with patch("artemis.tools.mobile.launch_app.find_package") as mock_find_package:
mock_find_package.return_value = None
outcome, _, _, _ = await _exec_manage_app(
action="launch",
app_name="NonExistentApp",
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error finding package for app: NonExistentApp" in outcome
# ==========================================
# SLIDE 8: wait_for_delay
# ==========================================
@pytest.mark.asyncio
async def test_wait_for_delay_happy_path(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_wait_for_delay(
time_in_ms=100,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Waited for 100ms successfully." in outcome
tests/unit/agents/test_flash_runner.py
+27
View File
@@ -18,7 +18,9 @@ from unittest.mock import Mock, patch
import pytest
from langchain_core.messages import HumanMessage, ToolMessage
from artemis.agents.explorer.constants import ASK_EXPLORER_DESCRIPTION
from artemis.agents.flash.runner import FlashRunner
from artemis.agents.validator.tool_declarations import ASK_EXPLORER_TOOL
from artemis.context import ArtemisContext
@@ -57,6 +59,31 @@ def test_flash_runner_tools_initialization(mock_context):
assert "report_failure_analysis" not in tool_names
def test_flash_runner_executor_follows_the_flash_explorer_tier(mock_context):
"""The executor is tagged as the Flash profile so ask_explorer follows flash_mode."""
with (
patch("artemis.controllers.unified_controller.get_driver"),
patch("artemis.agents.flash.runner.McpActionExecutor") as executor_cls,
):
runner = FlashRunner(mock_context, goal="Test Open Settings")
executor_cls.assert_called_once()
assert executor_cls.call_args.kwargs["agent_name"] == "flash"
assert runner.executor is executor_cls.return_value
def test_ask_explorer_declaration_is_tier_agnostic(mock_context):
"""Flash binds the shared ask_explorer contract: query + context_feedback, no tier."""
with patch("artemis.controllers.unified_controller.get_driver"):
runner = FlashRunner(mock_context, goal="Test Open Settings")
declaration = next(t for t in runner._get_tools() if t.name == "ask_explorer")
assert declaration is ASK_EXPLORER_TOOL
assert declaration.description == ASK_EXPLORER_DESCRIPTION
assert set(declaration.parameters["properties"]) == {"query", "context_feedback"}
assert declaration.parameters["required"] == ["query"]
def test_flash_runner_screenshot_pruning(mock_context):
"""Verify that earlier screenshot blocks are pruned, keeping only the latest."""
with patch("artemis.controllers.unified_controller.get_driver"):
tests/unit/agents/test_flash_runner_ledger.py
+333
View File
@@ -0,0 +1,333 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""FlashRunner over the session transcript ledger (Pro-identical history).
Covers: the ``video_analyzer`` availability gate (same as the Pro graph) and
its prompt segment, the unbounded-by-default turn limit, the Pro-shaped
observation tail (``# CURRENT OBSERVATION [T+mm:ss]`` header, screenshot,
``--- Visible UI Elements ---`` list), and the end-to-end loop: committed
turns carry text-only tool messages plus an ``--- Action Execution Result
(T+mm:ss) ---`` message, every recorded step of a multi-action turn is
registered on the committed turn, the final-turn tool restriction only
applies to bounded loops, and a no-tool-call turn is nudged, not terminated.
"""
import base64
import re
from unittest.mock import AsyncMock, Mock, patch
import pytest
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
from artemis.agents.flash.runner import FlashRunner
from artemis.agents.validator.tool_declarations import ToolExecutionResult
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.memory.transcript import (
EXECUTION_RESULT_MARKER,
PRO_UI_LIST_MARKER,
TranscriptLedger,
)
OBSERVATION_HEADER_RE = re.compile(r"^# CURRENT OBSERVATION \[T\+\d{2,}:\d{2}\]$")
RESULT_RE = re.compile(
rf"^{re.escape(EXECUTION_RESULT_MARKER)} \(T\+\d{{2,}}:\d{{2}}\) ---\nStatus: success$"
)
@pytest.fixture
def mock_context():
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = Mock()
mock_llm_cfg = Mock()
mock_llm_cfg.model = "gemini-2.5-flash"
mock_llm_cfg.temperature = 0.1
ctx.llm_config.get_agent.return_value = mock_llm_cfg
ctx.device = Mock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.data_engine = None
ctx.adb_client = None
ctx.driver = Mock()
return ctx
# ---------------------------------------------------------------------------
# video_analyzer availability (same gate as the Pro graph)
# ---------------------------------------------------------------------------
def test_video_analyzer_bound_only_when_recording_tools_enabled(mock_context):
with patch("artemis.controllers.unified_controller.get_driver"):
mock_context.execution_setup = Mock()
mock_context.execution_setup.video_recording_tools_enabled = True
names = [t.name for t in FlashRunner(mock_context, goal="g")._get_tools()]
assert "video_analyzer" in names
assert names[-1] == "report_task_status"
mock_context.execution_setup.video_recording_tools_enabled = False
names = [t.name for t in FlashRunner(mock_context, goal="g")._get_tools()]
assert "video_analyzer" not in names
# A Mock attribute (truthy but not True) must not enable the tool.
mock_context.execution_setup = Mock()
names = [t.name for t in FlashRunner(mock_context, goal="g")._get_tools()]
assert "video_analyzer" not in names
def test_recall_history_bound_only_with_a_data_engine_session(mock_context):
with patch("artemis.controllers.unified_controller.get_driver"):
# No DataEngine: nothing to search, the tool stays out (as in Pro).
names = [t.name for t in FlashRunner(mock_context, goal="g")._get_tools()]
assert "recall_history" not in names
mock_context.data_engine = Mock()
with patch("artemis.tools.history_recall._recall_config", return_value=None):
runner = FlashRunner(mock_context, goal="g")
tools = runner._get_tools()
names = [t.name for t in tools]
assert "recall_history" in names
declaration = next(t for t in tools if t.name == "recall_history")
assert set(declaration.parameters["properties"]) == {
"query",
"step_range",
"include_details",
"include_images",
"max_results",
}
assert declaration.parameters["required"] == ["query"]
assert "`recall_history`" in runner._render_system_prompt(tools)
def test_video_analyzer_prompt_segment_follows_availability(mock_context):
with patch("artemis.controllers.unified_controller.get_driver"):
runner = FlashRunner(mock_context, goal="g")
mock_context.execution_setup = Mock()
mock_context.execution_setup.video_recording_tools_enabled = True
with_video = runner._render_system_prompt(runner._get_tools())
assert "`video_analyzer`" in with_video
mock_context.execution_setup.video_recording_tools_enabled = False
without_video = runner._render_system_prompt(runner._get_tools())
assert "video_analyzer" not in without_video
# Session-clock / history teaching is unconditional.
assert "T+mm:ss" in without_video
assert "Action Execution Result" in without_video
# ---------------------------------------------------------------------------
# Turn limit: unbounded by default, explicit caps honoured
# ---------------------------------------------------------------------------
def test_turn_limit_semantics(mock_context):
with patch("artemis.controllers.unified_controller.get_driver"):
assert FlashRunner(mock_context, goal="g", max_turns=0).turn_limit is None
assert FlashRunner(mock_context, goal="g", max_turns=7).turn_limit == 7
with patch("artemis.agents.flash.runner.load_agent_config", side_effect=RuntimeError):
assert FlashRunner(mock_context, goal="g").turn_limit is None
# ---------------------------------------------------------------------------
# Observation tail: Pro shape with the session-relative header
# ---------------------------------------------------------------------------
def test_observation_tail_has_pro_shape(mock_context):
with patch("artemis.controllers.unified_controller.get_driver"):
runner = FlashRunner(mock_context, goal="Open Settings")
ledger = TranscriptLedger()
tail = runner._build_tail(ledger, 1, b"IMG", "[1] Settings")
texts = [b["text"] for b in tail.content if b["type"] == "text"]
assert texts[0] == "Your objective is: Open Settings"
assert OBSERVATION_HEADER_RE.match(texts[1])
assert texts[2] == "--- Current Screenshot ---"
assert texts[3] == f"{PRO_UI_LIST_MARKER}\n[1] Settings"
assert sum(1 for b in tail.content if b["type"] == "image_url") == 1
later = runner._build_tail(
ledger, 5, None, None, injected="[INJECTED] stop", notices=["nudge"], is_final=True
)
texts = [b["text"] for b in later.content]
assert OBSERVATION_HEADER_RE.match(texts[0])
assert "Your objective is" not in "".join(texts)
assert texts[1:] == [
"nudge",
"[INJECTED] stop",
"[WARNING] This is your final turn; only 'report_task_status' is available.",
]
# ---------------------------------------------------------------------------
# End-to-end loop over the ledger (mocked model / executor)
# ---------------------------------------------------------------------------
def _exec_result(tc_id, name, post_bytes, ui="[1] next"):
return ToolExecutionResult(
tool_call_id=tc_id,
tool_name=name,
status="success",
text_summary=f"{name} executed",
screenshot_bytes=post_bytes,
ui_elements_text=ui,
)
def _make_runner(mock_context, responses, max_turns=None):
runner = FlashRunner(mock_context, goal="Open Settings", max_turns=max_turns)
runner.summarizer = None
runner._init_llm = Mock(return_value=Mock())
# Snapshot each invocation: the runner appends to the same message list
# after the call, so the recorded call args would otherwise drift.
runner.calls = []
pending = list(responses)
async def _invoke(llm, tools, messages):
runner.calls.append({"tools": list(tools), "messages": list(messages)})
return pending.pop(0)
runner._invoke_model = AsyncMock(side_effect=_invoke)
runner.executor.execute = AsyncMock(
side_effect=lambda name, args, tc_id, state: _exec_result(
tc_id, name, f"IMG-{tc_id}".encode()
)
)
return runner
def _report(status="completed", tc_id="tc-report", **extra):
return AIMessage(
content="Done.",
tool_calls=[
{"name": "report_task_status", "args": {"status": status, **extra}, "id": tc_id}
],
)
_INITIAL_OBSERVATION = ("shot0.png", b"IMG0", "[1] Settings")
@pytest.mark.asyncio
async def test_run_builds_prompt_from_ledger_with_session_offsets(mock_context):
"""Turn 2 must see: the static system prefix, the committed turn 1 (its
tail, the AI message, text-only tool messages and an execution-result
message with a ``T+`` offset) and a fresh observation tail carrying the
last action's post screenshot — the Pro transcript shape, with no cap."""
responses = [
AIMessage(
content="I will tap then wait.",
tool_calls=[
{"name": "click", "args": {"target": [500, 600]}, "id": "tc1"},
{"name": "wait_for_delay", "args": {"seconds": 1}, "id": "tc2"},
],
),
_report(explanation="ok"),
]
with (
patch("artemis.controllers.unified_controller.get_driver"),
patch(
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
AsyncMock(return_value=_INITIAL_OBSERVATION),
),
):
runner = _make_runner(mock_context, responses, max_turns=0)
assert runner.turn_limit is None
result = await runner.run(State(initial_goal="Open Settings"))
assert result == {"status": "completed", "explanation": "ok"}
assert runner._invoke_model.await_count == 2
turn2_messages = runner.calls[1]["messages"]
assert isinstance(turn2_messages[0], SystemMessage)
assert "T+mm:ss" in turn2_messages[0].content
tail1 = turn2_messages[1]
assert isinstance(tail1, HumanMessage)
tail1_texts = [b["text"] for b in tail1.content if b["type"] == "text"]
assert tail1_texts[0] == "Your objective is: Open Settings"
assert OBSERVATION_HEADER_RE.match(tail1_texts[1])
assert isinstance(turn2_messages[2], AIMessage)
tool_msgs = [m for m in turn2_messages if isinstance(m, ToolMessage)]
assert [m.content for m in tool_msgs] == ["click executed", "wait_for_delay executed"]
result_msg = turn2_messages[5]
assert isinstance(result_msg, HumanMessage)
assert RESULT_RE.match(result_msg.content[0]["text"])
tail2 = turn2_messages[6]
tail2_texts = [b["text"] for b in tail2.content if b["type"] == "text"]
assert OBSERVATION_HEADER_RE.match(tail2_texts[0])
assert "Your objective is" not in "".join(tail2_texts)
images = [b for b in tail2.content if b["type"] == "image_url"]
assert len(images) == 1
assert images[0]["image_url"]["url"].endswith(base64.b64encode(b"IMG-tc2").decode())
assert tail2_texts[1] == "--- Current Screenshot ---"
assert tail2_texts[2] == f"{PRO_UI_LIST_MARKER}\n[1] next"
assert len(turn2_messages) == 7
# Both actions of the multi-action turn were registered on the committed turn.
ledger = mock_context.transcript_ledger
assert ledger.unchunked_turns()[0]["step_keys"] == ["tc1", "tc2"]
@pytest.mark.asyncio
async def test_run_final_turn_restricts_tools_and_fails_without_report(mock_context):
responses = [AIMessage(content="I give up.", tool_calls=[])]
with (
patch("artemis.controllers.unified_controller.get_driver"),
patch(
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
AsyncMock(return_value=_INITIAL_OBSERVATION),
),
):
runner = _make_runner(mock_context, responses, max_turns=1)
result = await runner.run(State(initial_goal="Open Settings"))
assert result == {"status": "failed", "explanation": "I give up."}
tools = runner.calls[0]["tools"]
assert [t.name for t in tools] == ["report_task_status"]
tail = runner.calls[0]["messages"][-1]
assert any("final turn" in b.get("text", "") for b in tail.content)
@pytest.mark.asyncio
async def test_run_no_tool_call_turn_is_nudged_not_terminated(mock_context):
responses = [AIMessage(content="thinking only", tool_calls=[]), _report()]
with (
patch("artemis.controllers.unified_controller.get_driver"),
patch(
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
AsyncMock(return_value=_INITIAL_OBSERVATION),
),
):
runner = _make_runner(mock_context, responses, max_turns=0)
result = await runner.run(State(initial_goal="Open Settings"))
assert result == {"status": "completed"}
turn2_messages = runner.calls[1]["messages"]
# Unbounded loop: no final-turn restriction was ever applied.
tools = runner.calls[1]["tools"]
assert "click" in [t.name for t in tools]
tail2_texts = [b["text"] for b in turn2_messages[-1].content if b["type"] == "text"]
assert any("did not call any tools" in t for t in tail2_texts)
# A helper-only / empty turn commits without an execution-result message.
assert not any(
isinstance(m, HumanMessage)
and any(EXECUTION_RESULT_MARKER in b.get("text", "") for b in m.content)
for m in turn2_messages
)
tests/unit/agents/test_flash_scrub_edge.py
+3 -9
View File
@@ -74,9 +74,7 @@ def _content_fingerprint(msg) -> str:
def _has_image(msg) -> bool:
return any(
isinstance(b, dict) and b.get("type") in ("image_url", "image") for b in msg.content
)
return any(isinstance(b, dict) and b.get("type") in ("image_url", "image") for b in msg.content)
def test_success_path_product_behind_the_scrub_edge(mock_context):
@@ -204,9 +202,7 @@ def test_race_failed_summary_becomes_unavailable_placeholder(mock_context):
_simulate_turns(compressor, messages, [_step(1), _step(2), _step(3)])
assert not _has_image(messages[0])
assert "[visual summary unavailable; evidence at DataEngine step 1]" in str(
messages[0].content
)
assert "[visual summary unavailable; evidence at DataEngine step 1]" in str(messages[0].content)
def test_freeze_invariant_bytes_never_change(mock_context):
@@ -232,9 +228,7 @@ def test_freeze_invariant_bytes_never_change(mock_context):
for idx in compressor._frozen:
fingerprint = _content_fingerprint(messages[idx])
if idx in frozen_fingerprints:
assert frozen_fingerprints[idx] == fingerprint, (
f"frozen message {idx} mutated"
)
assert frozen_fingerprints[idx] == fingerprint, f"frozen message {idx} mutated"
else:
frozen_fingerprints[idx] = fingerprint
tests/unit/agents/test_flash_step_summarizer.py
+1 -3
View File
@@ -551,9 +551,7 @@ async def test_raw_lens_call_meters_llm_usage_without_touching_context_base(mock
await summarizer.flush()
usage_calls = [
c
for c in engine.record_trace.call_args_list
if c.kwargs.get("name") == "llm_usage"
c for c in engine.record_trace.call_args_list if c.kwargs.get("name") == "llm_usage"
]
assert usage_calls, "the raw lens bypass must record an llm_usage trace"
payload = usage_calls[0].kwargs["payload"]
tests/unit/agents/test_historical_state_hint.py
+1 -3
View File
@@ -77,9 +77,7 @@ async def _run(steps, b64=B64_A, cfg=None):
builder = PromptBuilder()
component = HistoricalStateHintPromptComponent()
with patch("artemis.config.load_agent_config", return_value=cfg or _cfg()):
await component(
builder, MagicMock(), MagicMock(), latest_screenshot_b64=b64, steps=steps
)
await component(builder, MagicMock(), MagicMock(), latest_screenshot_b64=b64, steps=steps)
return [p for p in builder.human_parts if isinstance(p, str)]
tests/unit/agents/test_history_analyzer.py
+1 -3
View File
@@ -354,9 +354,7 @@ async def test_history_analyzer_integration_with_task_tree():
"artemis.agents.history_analyzer.history_analyzer.get_note_file_path",
return_value=mock_path,
),
patch(
"artemis.agents.history_analyzer.history_analyzer.build_history_for"
) as mock_build,
patch("artemis.agents.history_analyzer.history_analyzer.build_history_for") as mock_build,
):
mock_build.return_value = "Mocked operation history"
analyzer = HistoryAnalyzer(mock_ctx)
tests/unit/agents/test_operator.py
+34 -48
View File
@@ -414,7 +414,9 @@ async def test_operator_dynamic_prompt():
mock_llm.bind_tools.return_value = mock_llm
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(mock_ctx, prompt_components=[MockComponent()], transcript_config=LEGACY_TRANSCRIPT)
node = OperatorNode(
mock_ctx, prompt_components=[MockComponent()], transcript_config=LEGACY_TRANSCRIPT
)
node_update = await node(mock_state)
@@ -554,28 +556,6 @@ async def test_operator_history_compression():
node_update = await node(mock_state)
@pytest.mark.asyncio
async def test_operator_loop_detection():
from artemis.agents.operator.operator import OperatorNode
from artemis.context import ArtemisContext
from unittest.mock import MagicMock
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.execution_setup = None
mock_ctx.data_engine = None
mock_state = MagicMock()
mock_state.subagent_calls = ["ask_explorer", "ask_explorer", "ask_explorer"]
mock_state.initial_goal = "Test goal"
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
with pytest.raises(RuntimeError) as exc_info:
node_update = await node(mock_state)
assert "Infinite loop detected" in str(exc_info.value)
@pytest.mark.asyncio
async def test_operator_tracks_subagent_calls():
from artemis.agents.operator.operator import OperatorNode
@@ -630,7 +610,9 @@ async def test_operator_tracks_subagent_calls():
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT)
node = OperatorNode(
mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT
)
node_update = await node(mock_state)
update_dict = node_update
@@ -722,7 +704,9 @@ async def test_operator_defer_action_for_gathering():
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT)
node = OperatorNode(
mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT
)
node_update = await node(mock_state)
# Subagent tool should have been executed
@@ -908,7 +892,9 @@ async def test_operator_no_defer_for_note_updating():
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[mock_update_note_tool], transcript_config=LEGACY_TRANSCRIPT)
node = OperatorNode(
mock_ctx, tools=[mock_update_note_tool], transcript_config=LEGACY_TRANSCRIPT
)
node_update = await node(mock_state)
# The update_note tool should have been executed immediately
@@ -998,7 +984,9 @@ async def test_operator_defer_for_note_reading():
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[mock_read_note_tool], transcript_config=LEGACY_TRANSCRIPT)
node = OperatorNode(
mock_ctx, tools=[mock_read_note_tool], transcript_config=LEGACY_TRANSCRIPT
)
node_update = await node(mock_state)
# The read_note tool should have been executed
@@ -1086,7 +1074,9 @@ async def test_operator_no_defer_for_note_appending():
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[mock_append_note_tool], transcript_config=LEGACY_TRANSCRIPT)
node = OperatorNode(
mock_ctx, tools=[mock_append_note_tool], transcript_config=LEGACY_TRANSCRIPT
)
node_update = await node(mock_state)
# The append_note tool should have been executed immediately
@@ -1106,7 +1096,7 @@ async def test_operator_no_defer_for_note_appending():
async def test_operator_background_tasks_prompt_injection():
from artemis.agents.operator.operator import OperatorNode
from artemis.context import ArtemisContext
from artemis.tools.command_tool import _BACKGROUND_TASKS, BackgroundTask
from artemis.tools.command_tool import BackgroundTask, get_adb_task_registry
from unittest.mock import AsyncMock, MagicMock, patch
mock_ctx = MagicMock(spec=ArtemisContext)
@@ -1140,7 +1130,8 @@ async def test_operator_background_tasks_prompt_injection():
cwd="/data/local/tmp",
)
_BACKGROUND_TASKS["task_99999"] = bg_task
adb_registry = get_adb_task_registry(mock_ctx)
adb_registry.background["task_99999"] = bg_task
try:
captured_messages = []
@@ -1194,8 +1185,7 @@ async def test_operator_background_tasks_prompt_injection():
assert "term_xyz" in full_content
assert "Accumulated Output: 0 lines of logs" in full_content
finally:
if "task_99999" in _BACKGROUND_TASKS:
del _BACKGROUND_TASKS["task_99999"]
adb_registry.background.pop("task_99999", None)
@pytest.mark.asyncio
@@ -1225,11 +1215,16 @@ async def test_operator_task_plan_warning_prompt_injection():
mock_llm.ainvoke = AsyncMock(return_value=mock_response)
mock_llm.bind_tools.return_value = mock_llm
# Case 1: task_plan updated in the last two steps (using the "key" argument)
# Case 1: task_plan updated in the last step (using the "key" argument)
steps_updated = [
{
"step_id": "step_1",
"step_number": 1,
"tool_calls": [{"name": "click", "args": {"target": 1}}],
},
{
"step_id": "step_2",
"step_number": 2,
"tool_calls": [
{
"name": "update_note",
@@ -1237,11 +1232,6 @@ async def test_operator_task_plan_warning_prompt_injection():
}
],
},
{
"step_id": "step_2",
"step_number": 2,
"tool_calls": [{"name": "click", "args": {"target": 1}}],
},
]
captured_messages_1 = []
@@ -1278,12 +1268,9 @@ async def test_operator_task_plan_warning_prompt_injection():
elif isinstance(m.content, str):
full_content_1 += m.content + "\n"
assert (
"Reminder: You have not updated the task_plan for two consecutive"
" turns." not in full_content_1
)
assert "Reminder: your last turn executed a Turn-Ending Action" not in full_content_1
# Case 2: task_plan NOT updated in the last two steps
# Case 2: an action turn that did NOT update task_plan
steps_not_updated = [
{
"step_id": "step_1",
@@ -1294,6 +1281,7 @@ async def test_operator_task_plan_warning_prompt_injection():
"step_id": "step_2",
"step_number": 2,
"tool_calls": [{"name": "click", "args": {"target": 2}}],
"action_taken": {"action": "click", "target": 2},
},
]
@@ -1331,10 +1319,7 @@ async def test_operator_task_plan_warning_prompt_injection():
elif isinstance(m.content, str):
full_content_2 += m.content + "\n"
assert (
"Reminder: You have not updated the task_plan for two consecutive"
" turns." in full_content_2
)
assert "Reminder: your last turn executed a Turn-Ending Action" in full_content_2
@pytest.mark.asyncio
@@ -1399,7 +1384,8 @@ async def test_operator_tool_limit_exceeded_warning():
full_content += m.content + "\n"
assert (
"You did not execute any screen interaction actions in your last turn" in full_content
"your last turn used up its tool-call budget without a Turn-Ending Action"
in full_content
)
assert node_update.get("operator_tool_limit_exceeded") is False
tests/unit/agents/test_operator_prompts.py
+13 -3
View File
@@ -22,9 +22,19 @@ from artemis.agents.operator.prompts import apply_operator_prompt_contract
def test_operator_prompt_contract_matches_note_runtime_semantics():
prompt = apply_operator_prompt_contract(load_operator_prompts()["main_template"])
assert "write-through tools `update_note` and `append_note`" in prompt
assert "may accompany at most one Turn-Ending Action" in prompt
assert "`read_note`, `list_notes`, `save_note`, and `recall_history`" in prompt
assert "**Write-Through Memory Tools** (`save_note`, `update_note`, `append_note`)" in prompt
assert "May run alongside the turn's Turn-Ending Action (or Fast-Action Burst)" in prompt
# The two execution tiers and the incident workflow are advertised.
assert "**Vetted Single Action (default)**" in prompt
# The burst ceiling is a context variable filled in by the later template render.
assert (
"**Fast-Action Burst (time-critical only)**: two to {{ max_burst_actions }}"
" Turn-Ending Actions" in prompt
)
assert "--- Execution Incident (OPEN) ---" in prompt
assert "Failure Analyzer" not in prompt
assert "(`read_note`, `list_notes`, `recall_history`)" in prompt
assert "`save_note`, and `recall_history`" not in prompt
assert "Do NOT submit a Turn-Ending Action at the same time" not in prompt
assert "memory note tools (`read_note`, `list_notes`, `save_note`, `update_note`" not in prompt
tests/unit/agents/test_operator_transcript.py
+11 -9
View File
@@ -53,13 +53,17 @@ from artemis.memory.transcript import (
# SHA-256 of the legacy system message rendered by TemplatePromptComponent
# with the fixed inputs below. Originally recorded BEFORE the M2 template
# split; re-recorded once in M5 (2026-09-01) after the only intended template
# change since — removing the Short-Term Memory instruction block from
# operator.json (the <short_term_memory> relay was removed end to end per
# pro-context-memory-redesign §3.3). Any other drift here means the flag-off
# split; re-recorded in M5 (2026-09-01) after the Short-Term Memory block was
# removed, and again on 2026-09-02 when the Failure Analyzer was retired: the
# operator.json template gained the Time-Sensitive Tasks principle, the two
# execution tiers (vetted single action / fast-action burst), the Execution
# Incident workflow section and the Phase 3 burst-discipline rule; and again
# on 2026-09-02 after the prompt consolidation (every rule stated once, the
# transcript/legacy history-marker split, the recited tool budget, the
# recommended diagnosis triggers). Any other drift here means the flag-off
# path is no longer byte-identical.
GOLDEN_EMPTY_PLAN = "c45aec7d67069dad3a195534dd3c97dadb48454ca186a40eb471d5f9136589fa"
GOLDEN_SENTINEL_PLAN = "670580efd48f237c3257d996a814145fbef4ec0bb467aa4dfa6eb2db4a613826"
GOLDEN_EMPTY_PLAN = "a20046ff46058fa331fa3b8f2d3ffa4a018be567bf264588b9a29bc64e5b11b8"
GOLDEN_SENTINEL_PLAN = "8f4456524c8a964d1614d26d2fafab809bd5bd3f30334fd40498999e4b27638b"
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
@@ -253,9 +257,7 @@ async def test_transcript_actionless_turn_commits_without_validator_message():
ledger = ctx.transcript_ledger
assert ledger.turn_count == 1
assert not any(
EXECUTION_RESULT_MARKER in str(m.content) for m in ledger.active_messages
)
assert not any(EXECUTION_RESULT_MARKER in str(m.content) for m in ledger.active_messages)
@pytest.mark.asyncio
tests/unit/agents/test_plan_mutation_guardrails.py
-1
View File
@@ -141,7 +141,6 @@ def _guard_ctx(tmp_path, **setup_kwargs):
ctx.planner_task = None
ctx.last_validated_plan = None
ctx.pending_validated_plan = None
ctx.task_plan_content_before = None
ctx.pending_checkpoints = []
ctx.checkpoint_tasks = {}
return ctx
tests/unit/agents/test_prompt_assembly.py
+16 -61
View File
@@ -26,13 +26,6 @@ from artemis.agents.operator.prompts import (
apply_operator_prompt_contract,
)
from artemis.agents.prompt_assembly import gate_segment, render_tool_enum
from artemis.agents.validator.failure_analyzer import (
FailureAnalysisStrategy,
FailureAnalyzer,
PixelTargetDisappearedStrategy,
TargetDisappearedStrategy,
)
from artemis.agents.validator.validator import ValidationErrorCategory
REPO_ROOT = Path(__file__).resolve().parents[3]
@@ -115,12 +108,15 @@ def test_removed_swipe_takes_its_teaching_along():
def test_removed_wait_for_delay_keeps_interval_line_coherent():
"""Only the wait_for_delay parenthetical goes; the SOP grammar line stays."""
"""Only the wait_for_delay parenthetical goes; the SOP pacing step stays."""
template = load_operator_prompts()["main_template"]
out = apply_operator_prompt_contract(
template, available_tools=OPERATOR_PROMPT_TOOLSET - {"wait_for_delay"}
)
assert "`Interval: <Interval>`: Declared polling cadence." in out
assert (
"Adhere strictly to the exact duration declared in `<Interval>`, then execute"
" a refresh action" in out
)
assert "Loading & Transitions" not in out
@@ -131,12 +127,10 @@ def test_reduced_enums_stay_well_formed():
available_tools=OPERATOR_PROMPT_TOOLSET - {"manage_app", "wait_for_delay"},
)
assert (
"Physical device actions (`click`, `input_text`, `swipe`, `press_key`,"
" `long_press`)" in out
"Physical device actions (`click`, `input_text`, `swipe`, `press_key`, `long_press`)" in out
)
assert (
"Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`, or"
" `press_key`)" in out
"Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`, or `press_key`)" in out
)
@@ -152,20 +146,20 @@ def test_full_set_enum_slots_render_verbatim():
"Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`,"
" `press_key`, `manage_app`, or `wait_for_delay`)" in out
)
assert (
"(`ask_explorer`, `ask_diagnoser`, `video_analyzer`, `run_adb_command`,"
" `manage_task`, `analyze_task_output`, `read_note`, `list_notes`,"
" `save_note`, and `recall_history`)" in out
)
assert "Helper/Subagent tools (`ask_explorer`, `ask_diagnoser`, `video_analyzer`)" in out
assert "ADB/task tools (`run_adb_command`, `manage_task`)" in out
assert "(`read_note`, `list_notes`, `recall_history`)" in out
# save_note is a write-through tool, never a result-dependent one; the on-demand
# analyze_task_output is not advertised.
assert "`save_note`, and" not in out
assert "analyze_task_output" not in out
# --- Flash prompt assembly -----------------------------------------------------------
def _render_flash(available_tools: frozenset[str]) -> str:
template_text = (REPO_ROOT / "artemis/agents/flash/flash_runner.md").read_text(
encoding="utf-8"
)
template_text = (REPO_ROOT / "artemis/agents/flash/flash_runner.md").read_text(encoding="utf-8")
return Template(template_text).render(goal="test goal", available_tools=available_tools)
@@ -184,7 +178,7 @@ def test_flash_without_manage_app_drops_app_launching():
assert "**App Launching**" not in out
assert not _tool_refs(out, "manage_app")
# The neighbouring numbered rule survives.
assert "**Screenshot Pruning Awareness**" in out
assert "**Do not determine element coordinates based on guesswork.**" in out
def test_flash_without_wait_for_delay_drops_timed_waiting():
@@ -192,42 +186,3 @@ def test_flash_without_wait_for_delay_drops_timed_waiting():
assert "**Timed Waiting & Transitions**" not in out
assert not _tool_refs(out, "wait_for_delay")
assert "**State Drift & Progression Tolerance**" in out
# --- FailureAnalyzer strategy gating -------------------------------------------------
def _analyzer() -> FailureAnalyzer:
return FailureAnalyzer.__new__(FailureAnalyzer) # _select_strategy needs no ctx
def test_strategy_selection_unchanged_without_gating():
analyzer = _analyzer()
assert isinstance(
analyzer._select_strategy(ValidationErrorCategory.TARGET_DISAPPEARED),
TargetDisappearedStrategy,
)
assert isinstance(
analyzer._select_strategy(ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED),
PixelTargetDisappearedStrategy,
)
def test_strategy_falls_back_when_required_tool_missing():
"""Strategy + prompt are one unit: no click_sequence, no disappeared-prompts."""
analyzer = _analyzer()
strategy = analyzer._select_strategy(
ValidationErrorCategory.TARGET_DISAPPEARED,
available_tools=frozenset({"click", "swipe"}),
)
assert type(strategy) is FailureAnalysisStrategy
assert strategy.get_prompt_template_name() == "failure_analyzer.md"
def test_strategy_kept_when_required_tool_present():
analyzer = _analyzer()
strategy = analyzer._select_strategy(
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
available_tools=frozenset({"click_sequence"}),
)
assert isinstance(strategy, PixelTargetDisappearedStrategy)
tests/unit/agents/test_summarizer.py
+1 -3
View File
@@ -136,9 +136,7 @@ async def test_summarizer_falls_back_to_structured_decisions(mock_context):
kwargs = mock_context.step_memory.dispatch.call_args.kwargs
assert kwargs["action_name"] == "swipe"
assert kwargs["action_args"]["coordinates"] == [1, 2, 3, 4]
assert kwargs["action_args"]["additional_actions"] == [
{"action": "tap", "coordinates": [9, 9]}
]
assert kwargs["action_args"]["additional_actions"] == [{"action": "tap", "coordinates": [9, 9]}]
@pytest.mark.asyncio
tests/unit/agents/test_validator.py
+178 -154
View File
@@ -27,7 +27,7 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch
from langchain_core.messages import AIMessage
import pytest
from artemis.agents.validator.failure_analyzer import ValidationErrorCategory
from artemis.agents.validator.categories import ValidationErrorCategory
from artemis.agents.validator.validator import ValidatorNode
from artemis.context import ArtemisContext
from artemis.mcp.action_types import ActionCode, ActionResult
@@ -39,10 +39,12 @@ class DummyState:
structured_decisions,
current_step_id=None,
latest_screenshot=None,
open_incident=None,
):
self.structured_decisions = structured_decisions
self.current_step_id = current_step_id
self.latest_screenshot = latest_screenshot
self.open_incident = open_incident
class FakeActionSession:
@@ -143,10 +145,9 @@ async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
@pytest.mark.asyncio
async def test_validator_failure_analysis(mock_mcp, mock_context, temp_screenshot):
"""Test that ValidatorNode triggers failure analysis on error and handles
cannot_fix via tool.
"""
async def test_validator_exec_error_opens_incident(mock_mcp, mock_context, temp_screenshot):
"""A vetted single action that the device rejects is retried once, then an
execution incident is opened for the Operator (no repair agent)."""
mock_mcp.action_handler = lambda name, args: (
(False, "Error: Element not found") if name == "click" else (True, "")
)
@@ -155,80 +156,168 @@ async def test_validator_failure_analysis(mock_mcp, mock_context, temp_screensho
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Analysis: Coordinates are likely wrong.",
}
with patch("artemis.utils.image_diff.check_ui_change", return_value=False):
result = await node(state)
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
assert report["status"] == "failed"
assert report["burst"] is False
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "tap"
assert report["execution"][0]["attempts"] == [
"Error: Element not found",
"Error: Element not found",
]
assert report["execution"][0]["repair"] == "Analysis: Coordinates are likely wrong."
assert "repair" not in report["execution"][0]
incident = report["incident"]
assert incident["kind"] == "exec_error"
assert incident["category"] == "general"
assert incident["reason"] == "Error: Element not found"
assert incident["consecutive_failures"] == 1
assert incident["burst_size"] == 1
assert incident["action_description"] == "Tapped element at [105, 205]"
# The same record rides in graph state for the Operator prompt.
assert result["open_incident"] == incident
@pytest.mark.asyncio
async def test_validator_repair_success(mock_mcp, mock_context, temp_screenshot):
"""Test that ValidatorNode terminates successfully and clears remaining actions
after successful repair.
"""
tap_count = 0
async def test_validator_burst_skips_safety_net_and_aborts_on_first_failure(
mock_mcp, mock_context, temp_screenshot
):
"""Two or more actions form a fast-action burst: no precondition gate, a
single dispatch per member, and the first failure aborts the rest."""
calls = []
def handler(name, args):
nonlocal tap_count
if name == "click":
tap_count += 1
if tap_count <= 2: # First action fails on both validator attempts
return False, "Error: Element not found"
calls.append(name)
if name == "click" and len([c for c in calls if c == "click"]) == 2:
return False, "Error: tap rejected"
return True, ""
mock_mcp.action_handler = handler
# Two actions planned initially
decisions = json.dumps(
[
{"action": "tap", "coordinates": [105, 205]},
{"action": "tap", "coordinates": [205, 305]},
{"action": "press_key", "keycode": "BACK"},
]
)
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch(
"artemis.agents.validator.execution_loop._run_precondition_gate",
new_callable=AsyncMock,
) as mock_gate,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
result = await node(state)
mock_gate.assert_not_called()
report = result["last_execution_result"]
assert report["burst"] is True
assert report["status"] == "failed"
assert len(report["execution"]) == 3
# First member executed cleanly.
assert "attempts" not in report["execution"][0]
# Second member failed after exactly one dispatch (bursts never retry).
assert report["execution"][1]["attempts"] == ["Error: tap rejected"]
# Third member never fired.
assert report["execution"][2]["attempts"] == ["Skipped (burst aborted)"]
assert len(mock_mcp.calls_for("click")) == 2
assert mock_mcp.calls_for("press_key") == []
incident = report["incident"]
assert incident["kind"] == "exec_error"
assert incident["action_index"] == 1
assert incident["burst_size"] == 3
assert result["open_incident"] == incident
@pytest.mark.asyncio
async def test_validator_burst_success_executes_every_member(
mock_mcp, mock_context, temp_screenshot
):
decisions = json.dumps(
[
{"action": "tap", "coordinates": [105, 205]},
{"action": "wait_for_delay", "time_in_ms": 1},
{"action": "tap", "coordinates": [205, 305]},
]
)
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
mock_analyze.return_value = {
"status": "fixed",
"analysis": "Fixed by updating actions",
}
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
result = await node(state)
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
assert len(report["execution"]) == 1 # Only the first failed action is in report
assert report["status"] == "success"
assert report["burst"] is True
assert report["incident"] is None
assert result["open_incident"] is None
assert result["last_closed_incident"] is None
assert len(mock_mcp.calls_for("click")) == 2
assert report["execution"][0]["coordinates"] == [105, 205]
assert report["execution"][0]["attempts"] == [
"Error: Element not found",
"Error: Element not found",
]
assert report["execution"][0]["repair"] == "Fixed by updating actions"
@pytest.mark.asyncio
async def test_validator_success_closes_open_incident(mock_mcp, mock_context, temp_screenshot):
"""A successful turn closes whatever incident was still open."""
previous = {
"kind": "safety_net",
"category": "target_disappeared",
"reason": "gone",
"action": {"action": "tap"},
"action_description": "Tapped element",
"consecutive_failures": 2,
}
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(
structured_decisions=decisions,
latest_screenshot=temp_screenshot,
open_incident=previous,
)
node = ValidatorNode(mock_context)
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
result = await node(state)
assert result["last_execution_result"]["status"] == "success"
assert result["open_incident"] is None
# The closed record is handed over once so the Operator settles the intent.
assert result["last_closed_incident"]["kind"] == "safety_net"
assert "closed_at_step" in result["last_closed_incident"]
@pytest.mark.asyncio
async def test_validator_consecutive_failures_escalate_incident(
mock_mcp, mock_context, temp_screenshot
):
"""Failing again while an incident is open continues its failure count."""
previous = {
"kind": "exec_error",
"category": "general",
"reason": "Error: Element not found",
"action": {"action": "tap"},
"action_description": "Tapped element",
"consecutive_failures": 2,
}
mock_mcp.action_handler = lambda name, args: (False, "Error: Element not found")
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(
structured_decisions=decisions,
latest_screenshot=temp_screenshot,
open_incident=previous,
)
node = ValidatorNode(mock_context)
with patch("artemis.utils.image_diff.check_ui_change", return_value=False):
result = await node(state)
assert result["open_incident"]["consecutive_failures"] == 3
@pytest.mark.asyncio
@@ -271,7 +360,7 @@ async def test_validator_focus_and_clear_text_no_ui_change(mock_mcp, mock_contex
@pytest.mark.asyncio
async def test_validator_silent_failure_treated_as_success(mock_mcp, mock_context, temp_screenshot):
"""Test that silent failure (exec succeeds but no UI change) is treated as
success and does not trigger FailureAnalyzer.
success and does not open an execution incident.
"""
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -281,13 +370,12 @@ async def test_validator_silent_failure_treated_as_success(mock_mcp, mock_contex
with (
patch("artemis.agents.validator.validator.VALIDATOR_POLL_TIMEOUT", 0.1),
patch("artemis.agents.validator.validator.VALIDATOR_POLL_INTERVAL", 0.01),
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False), # No UI change
):
result = await node(state)
# Assert FailureAnalyzer was NOT called
mock_analyze.assert_not_called()
# No incident is opened for a silent (no-UI-change) success.
assert result["open_incident"] is None
# Assert the action is considered executed successfully
assert "last_execution_result" in result
@@ -299,13 +387,11 @@ async def test_validator_silent_failure_treated_as_success(mock_mcp, mock_contex
@pytest.mark.asyncio
async def test_validator_records_skipped_actions_on_cannot_fix(
async def test_validator_burst_first_member_failure_marks_rest_skipped(
mock_mcp, mock_context, temp_screenshot
):
"""Test that ValidatorNode marks unexecuted actions as Skipped when repair fails."""
mock_mcp.action_handler = lambda name, args: (False, "Error: Connection lost")
# Two actions planned
decisions = json.dumps(
[
{"action": "tap", "coordinates": [100, 200]},
@@ -315,42 +401,27 @@ async def test_validator_records_skipped_actions_on_cannot_fix(
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Cannot fix: System crashed.",
}
with patch("artemis.utils.image_diff.check_ui_change", return_value=False):
result = await node(state)
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
assert len(report["execution"]) == 2
# First action failed
assert report["execution"][0]["action"] == "tap"
assert report["execution"][0]["attempts"] == [
"Error: Connection lost",
"Error: Connection lost",
]
assert report["execution"][0]["repair"] == "Cannot fix: System crashed."
# Second action was skipped
# A burst member is dispatched exactly once.
assert report["execution"][0]["attempts"] == ["Error: Connection lost"]
assert "repair" not in report["execution"][0]
assert report["execution"][1]["action"] == "press_key"
assert report["execution"][1]["attempts"] == ["Skipped"]
assert report["execution"][1]["attempts"] == ["Skipped (burst aborted)"]
assert report["incident"]["action_index"] == 0
assert report["incident"]["burst_size"] == 2
@pytest.mark.asyncio
async def test_validator_pre_execution_validation_and_repair(
async def test_validator_pre_execution_validation_opens_incident(
mock_mcp, mock_context, temp_screenshot
):
"""Test that pre-execution validation fails when element changes, and is
repaired by FailureAnalyzer.
"""Test that pre-execution validation fails when the element changes, and the
interception is handed to the Operator as an execution incident.
"""
mock_mcp.hierarchy = [
{
@@ -376,7 +447,6 @@ async def test_validator_pre_execution_validation_and_repair(
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch.object(
ValidatorNode,
@@ -389,29 +459,20 @@ async def test_validator_pre_execution_validation_and_repair(
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
"Pixel check failed",
)
mock_analyze.return_value = {
"status": "fixed",
"analysis": ("Pre-exec failure matched and resolved by switching to Sign Up button."),
}
result = await node(state)
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert "Pre-execution validation failed" in args[2]
assert "Login" in args[2]
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_OCCUPIED
incident = result["last_execution_result"]["incident"]
assert "Pre-execution validation failed" in incident["reason"]
assert "Login" in incident["reason"]
assert incident["category"] == ValidationErrorCategory.TARGET_OCCUPIED.value
assert "last_execution_result" in result
report = result["last_execution_result"]
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "tap"
assert "Pre-execution validation failed" in report["execution"][0]["attempts"][0]
assert (
report["execution"][0]["repair"]
== "Pre-exec failure matched and resolved by switching to Sign Up"
" button."
)
assert report["incident"]["kind"] == "safety_net"
assert report["status"] == "failed"
@pytest.mark.asyncio
@@ -443,7 +504,7 @@ async def test_validator_pre_execution_validation_self_healing(
node = ValidatorNode(mock_context)
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
await node(state)
result = await node(state)
# Center of [100, 220][300, 320] is pixel [200, 270]; the healed coordinates
# travel to the canonical click tool normalized to 0-1000 on a 1080x2400 screen:
@@ -487,7 +548,6 @@ async def test_validator_pre_execution_validation_anonymous_occupant(
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch.object(
ValidatorNode,
@@ -500,19 +560,16 @@ async def test_validator_pre_execution_validation_anonymous_occupant(
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
"Pixel check failed",
)
mock_analyze.return_value = {
"status": "fixed",
"analysis": "Pre-exec failure bypassed.",
}
await node(state)
result = await node(state)
# Assert that the validator correctly classified this as TARGET_OCCUPIED
# (due to the clickable anonymous view blocking the click)
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_OCCUPIED
assert "occupied/intercepted by a different element: interactive anonymous element" in args[2]
incident = result["last_execution_result"]["incident"]
assert incident["category"] == ValidationErrorCategory.TARGET_OCCUPIED.value
assert (
"occupied/intercepted by a different element: interactive anonymous element"
in incident["reason"]
)
@pytest.mark.asyncio
@@ -558,7 +615,7 @@ async def test_validator_pixel_validation_success(mock_mcp, mock_context, temp_s
@pytest.mark.asyncio
async def test_validator_pixel_validation_failure(mock_mcp, mock_context, temp_screenshot):
"""Test that the pixel safety net fails and triggers FailureAnalyzer
"""Test that the pixel safety net fails and opens an execution incident
when Gemini reports target is missing.
"""
decisions = json.dumps([{"action": "tap", "coordinates": [100, 200]}])
@@ -588,20 +645,13 @@ async def test_validator_pixel_validation_failure(mock_mcp, mock_context, temp_s
),
patch("artemis.agents.validator.validator.get_llm", return_value=mock_llm),
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
):
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Pixel safety net flagged missing target.",
}
node = ValidatorNode(mock_context)
await node(state)
result = await node(state)
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert "Pixel-level validation failed" in args[2]
assert kwargs.get("error_category") == ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED
incident = result["last_execution_result"]["incident"]
assert "Pixel-level validation failed" in incident["reason"]
assert incident["category"] == ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED.value
@pytest.mark.asyncio
@@ -639,14 +689,8 @@ async def test_validator_launch_app_failure_no_retry(mock_mcp, mock_context, tem
with (
patch("artemis.agents.validator.validator.VALIDATOR_POLL_TIMEOUT", 0.05),
patch("artemis.agents.validator.validator.VALIDATOR_POLL_INTERVAL", 0.01),
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Cannot launch app.",
}
node = ValidatorNode(mock_context)
result = await node(state)
@@ -693,7 +737,6 @@ async def test_validator_pre_execution_validation_disappeared_not_shifted(
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch.object(
ValidatorNode,
@@ -706,18 +749,12 @@ async def test_validator_pre_execution_validation_disappeared_not_shifted(
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
"Pixel check failed",
)
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Sign Up button disappeared.",
}
await node(state)
result = await node(state)
# Assert that the validator correctly classified this as TARGET_DISAPPEARED
# (since size mismatch prevents shift matching)
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_DISAPPEARED
incident = result["last_execution_result"]["incident"]
assert incident["category"] == ValidationErrorCategory.TARGET_DISAPPEARED.value
@pytest.mark.asyncio
@@ -800,7 +837,6 @@ async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_byp
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch.object(
ValidatorNode,
@@ -814,18 +850,12 @@ async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_byp
ValidationErrorCategory.PIXEL_BYPASSED,
"",
)
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": ("XML validation failed and pixel bypass did not override."),
}
result = await node(state)
await node(state)
# FailureAnalyzer SHOULD be called because XML validation failed
# An incident SHOULD be opened because XML validation failed
# and PIXEL_BYPASSED did not override it
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert "Pre-execution validation failed" in args[2]
incident = result["last_execution_result"]["incident"]
assert "Pre-execution validation failed" in incident["reason"]
@pytest.mark.asyncio
@@ -845,14 +875,8 @@ async def test_validator_failure_screenshot_mcp_error_handled_safely(
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Action failed and cannot fix.",
}
result = await node(state)
assert "last_execution_result" in result
tests/unit/agents/test_video_blackboard.py
+4 -12
View File
@@ -209,9 +209,7 @@ async def test_video_coordinator_retries_transient_failures():
async def test_audio_analysis_reuses_exact_completed_coverage(analyzer_context):
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
analyzer = VideoAnalyzer(analyzer_context)
claim = analyzer.blackboard.claim_segment(
2.0, 6.0, "hear notification", modality="audio"
)
claim = analyzer.blackboard.claim_segment(2.0, 6.0, "hear notification", modality="audio")
analyzer.blackboard.complete_segment(
2.0,
6.0,
@@ -222,12 +220,8 @@ async def test_audio_analysis_reuses_exact_completed_coverage(analyzer_context):
modality="audio",
)
with patch(
"artemis.agents.video_analyzer.video_analyzer.get_controller"
) as controller:
result = await analyzer.exec_analyze_audio_only(
2.0, 6.0, "hear notification"
)
with patch("artemis.agents.video_analyzer.video_analyzer.get_controller") as controller:
result = await analyzer.exec_analyze_audio_only(2.0, 6.0, "hear notification")
controller.assert_not_called()
assert result.startswith("CACHED AUDIO ANALYSIS:")
@@ -272,9 +266,7 @@ def test_session_deletion_removes_video_blackboard_rows(tmp_path):
engine.record_video_start(video_id, "device", recording)
board = VideoBlackboard(f"video:{video_id}", db_path=engine.storage.db_path)
board.add_observation(
{"start": 0.0, "end": 1.0, "target": "cleanup", "summary": "evidence"}
)
board.add_observation({"start": 0.0, "end": 1.0, "target": "cleanup", "summary": "evidence"})
assert board.list_observations()
engine.storage.delete_session(session_id)
tests/unit/agents/test_video_reliability.py
+1 -3
View File
@@ -138,9 +138,7 @@ async def test_native_chunk_exhaustion_commits_universal_fallback(tmp_path):
duration_seconds=5.0,
warning=None,
)
controller = SimpleNamespace(
extract_segment_metadata=AsyncMock(return_value=extracted)
)
controller = SimpleNamespace(extract_segment_metadata=AsyncMock(return_value=extracted))
fallback_text = "[from 0.0s to 5.0s] Summary: fallback recovered Analysis: grounded"
with (
tests/unit/core/test_adb_keys.py
+7 -21
View File
@@ -51,9 +51,7 @@ def test_inspect_adb_keys_not_exist(monkeypatch, tmp_path):
"""When keys do not exist yet, they are considered valid (will auto-generate on first connect)."""
priv = tmp_path / "adbkey"
pub = tmp_path / "adbkey.pub"
monkeypatch.setattr(
"artemis.core.diagnostics.adb_keys.get_adb_key_paths", lambda: (priv, pub)
)
monkeypatch.setattr("artemis.core.diagnostics.adb_keys.get_adb_key_paths", lambda: (priv, pub))
status = inspect_adb_keys()
assert status.is_valid is True
@@ -68,9 +66,7 @@ def test_inspect_adb_keys_zero_bytes(monkeypatch, tmp_path):
priv.write_text("")
pub.write_text("")
monkeypatch.setattr(
"artemis.core.diagnostics.adb_keys.get_adb_key_paths", lambda: (priv, pub)
)
monkeypatch.setattr("artemis.core.diagnostics.adb_keys.get_adb_key_paths", lambda: (priv, pub))
status = inspect_adb_keys()
assert status.is_valid is False
@@ -87,9 +83,7 @@ def test_inspect_adb_keys_invalid_pem_header(monkeypatch, tmp_path):
priv.write_text("GARBAGE DATA WITHOUT PEM HEADER " * 10)
pub.write_text("some pubkey")
monkeypatch.setattr(
"artemis.core.diagnostics.adb_keys.get_adb_key_paths", lambda: (priv, pub)
)
monkeypatch.setattr("artemis.core.diagnostics.adb_keys.get_adb_key_paths", lambda: (priv, pub))
status = inspect_adb_keys()
assert status.is_valid is False
@@ -106,9 +100,7 @@ def test_inspect_adb_keys_valid(monkeypatch, tmp_path):
)
pub.write_text("ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC... user@host")
monkeypatch.setattr(
"artemis.core.diagnostics.adb_keys.get_adb_key_paths", lambda: (priv, pub)
)
monkeypatch.setattr("artemis.core.diagnostics.adb_keys.get_adb_key_paths", lambda: (priv, pub))
status = inspect_adb_keys()
assert status.is_valid is True
@@ -127,9 +119,7 @@ def test_heal_adb_keys_healthy_no_op(monkeypatch, tmp_path):
)
pub.write_text("ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC... user@host")
monkeypatch.setattr(
"artemis.core.diagnostics.adb_keys.get_adb_key_paths", lambda: (priv, pub)
)
monkeypatch.setattr("artemis.core.diagnostics.adb_keys.get_adb_key_paths", lambda: (priv, pub))
res = heal_adb_keys(force=False)
assert res["success"] is True
@@ -143,9 +133,7 @@ def test_heal_adb_keys_corrupted(monkeypatch, tmp_path):
priv.write_text("")
pub.write_text("")
monkeypatch.setattr(
"artemis.core.diagnostics.adb_keys.get_adb_key_paths", lambda: (priv, pub)
)
monkeypatch.setattr("artemis.core.diagnostics.adb_keys.get_adb_key_paths", lambda: (priv, pub))
with patch("subprocess.run") as mock_run:
mock_run.return_value.returncode = 0
@@ -163,9 +151,7 @@ async def test_adb_probe_detects_corrupted_key_with_unauthorized_device(monkeypa
priv = tmp_path / "adbkey"
pub = tmp_path / "adbkey.pub"
priv.write_text("")
monkeypatch.setattr(
"artemis.core.diagnostics.adb_keys.get_adb_key_paths", lambda: (priv, pub)
)
monkeypatch.setattr("artemis.core.diagnostics.adb_keys.get_adb_key_paths", lambda: (priv, pub))
probe = AdbDeviceProbe()
monkeypatch.setattr(probe, "_locate_adb", lambda: "/usr/bin/adb")
tests/unit/core/test_diagnostics.py
+4 -13
View File
@@ -142,9 +142,7 @@ def test_adb_probe_parses_device_lock_state(policy_output, trust_output, expecte
def test_modern_unlock_state_is_not_overridden_by_legacy_fields():
policy = (
"KeyguardServiceDelegate\n"
" showing=false\n"
"mShowingLockscreen=true mKeyguardOccluded=false"
"KeyguardServiceDelegate\n showing=false\nmShowingLockscreen=true mKeyguardOccluded=false"
)
trust = 'User "Owner" (current): deviceLocked=0'
@@ -167,9 +165,7 @@ async def test_positive_lock_state_requires_confirmation(monkeypatch):
async def test_dashboard_reuses_recent_confirmed_state_on_one_timeout(monkeypatch):
probe = AdbDeviceProbe()
confirmed_probe = AsyncMock(side_effect=[False, None])
monkeypatch.setattr(
probe, "_get_confirmed_device_lock_state", confirmed_probe
)
monkeypatch.setattr(probe, "_get_confirmed_device_lock_state", confirmed_probe)
assert await probe._get_dashboard_lock_state("adb", "device-1") is False
assert await probe._get_dashboard_lock_state("adb", "device-1") is False
@@ -265,9 +261,7 @@ async def test_invalidation_prevents_in_flight_report_from_becoming_shared_cache
@pytest.mark.asyncio
async def test_submission_probe_skips_full_device_enrichment(monkeypatch):
probe = AdbDeviceProbe(target_serial="device-2")
get_states = AsyncMock(
return_value=[("device-1", "device"), ("device-2", "device")]
)
get_states = AsyncMock(return_value=[("device-1", "device"), ("device-2", "device")])
get_lock_state = AsyncMock(return_value=False)
full_probe = AsyncMock()
monkeypatch.setattr(
@@ -283,9 +277,7 @@ async def test_submission_probe_skips_full_device_enrichment(monkeypatch):
assert result.summary == "Connected"
assert result.metadata["submission_probe"] is True
get_states.assert_awaited_once_with("adb")
get_lock_state.assert_awaited_once_with(
"adb", "device-2", timeout_seconds=1.0
)
get_lock_state.assert_awaited_once_with("adb", "device-2", timeout_seconds=1.0)
full_probe.assert_not_awaited()
@@ -361,7 +353,6 @@ async def test_adb_probe_prefers_unlocked_device_when_one_is_locked(monkeypatch)
assert result.metadata["active_device"]["is_locked"] is False
@pytest.mark.asyncio
async def test_llm_credentials_probe_structure():
"""Verify LLMCredentialsProbe returns correct category and schema."""
tests/unit/data_engine/test_end_to_end_thoughts.py
+1 -3
View File
@@ -144,9 +144,7 @@ async def test_end_to_end_thoughts_non_duplication(temp_workspace):
# 8. Build history and check for duplications
plan = "- [ ] Search for books"
history = build_plan_and_history(
plan, steps, "default", last_n_detailed=1, for_failure_analyzer=True
)
history = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
print("\n--- Generated History Output ---")
print(history)
tests/unit/data_engine/test_engine.py
-1
View File
@@ -380,7 +380,6 @@ def test_update_step_summary_includes_step_number_in_sse(tmp_path):
assert last_update["step_number"] == 1
def _foreground_engine(tmp_path):
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
tests/unit/data_engine/test_latency_optimizations_deep.py
+6 -85
View File
@@ -391,10 +391,6 @@ async def test_validator_pre_execution_loop_reverted_to_exact_safety_contract(
"_validate_action_precondition_pixel",
new_callable=AsyncMock,
) as mock_pixel,
patch(
"artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze",
new_callable=AsyncMock,
) as mock_analyze,
):
mock_get_screen.return_value = (
base64.b64encode(b"dummy_bytes_here").decode("utf-8"),
@@ -413,93 +409,18 @@ async def test_validator_pre_execution_loop_reverted_to_exact_safety_contract(
"VLM target missing",
)
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Unrepairable drift",
}
result = await node(state)
# Verify both XML check and VLM check were called (reverted double-check contract)
mock_xml.assert_called_once()
mock_pixel.assert_called_once()
# Verify FailureAnalyzer received the EXACT XML error (TARGET_OCCUPIED), NOT the VLM error
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_OCCUPIED
assert "Occupied by Sign Up button" in args[2]
@pytest.mark.asyncio
async def test_failure_analyzer_prune_intermediate_screenshots(mock_artemis_ctx):
"""Verify FailureAnalyzer._prune_intermediate_screenshots keeps initial reference images and the latest live image, while pruning intermediate images."""
from google.genai import types
from artemis.agents.validator.failure_analyzer import FailureAnalyzer
analyzer = FailureAnalyzer(mock_artemis_ctx)
# Build simulated contents history:
# 0: User (Initial reference screenshots + prompt)
# 1: Model
# 2: User (Intermediate tool results)
# 3: User (Intermediate observation screenshot - SHOULD BE PRUNED)
# 4: Model
# 5: User (Intermediate tool results 2)
# 6: User (Latest observation screenshot - SHOULD BE PRESERVED)
contents = [
types.Content(
role="user",
parts=[
types.Part.from_text(text="Reference screenshot 1"),
types.Part.from_bytes(data=b"initial_pre_image_bytes", mime_type="image/jpeg"),
types.Part.from_bytes(data=b"initial_post_image_bytes", mime_type="image/jpeg"),
],
),
types.Content(role="model", parts=[types.Part.from_text(text="Clicked something")]),
types.Content(
role="user",
parts=[types.Part.from_text(text="Result text")],
),
types.Content(
role="user",
parts=[
types.Part.from_text(text="Intermediate observation"),
types.Part.from_bytes(
data=b"intermediate_image_bytes_to_prune",
mime_type="image/jpeg",
),
],
),
types.Content(role="model", parts=[types.Part.from_text(text="Clicking again")]),
types.Content(
role="user",
parts=[types.Part.from_text(text="Result text 2")],
),
types.Content(
role="user",
parts=[
types.Part.from_text(text="Latest live observation"),
types.Part.from_bytes(
data=b"latest_live_image_bytes_to_keep",
mime_type="image/jpeg",
),
],
),
]
analyzer._prune_intermediate_screenshots(contents)
# 1. Verify index 0 images are untouched
assert contents[0].parts[1].inline_data.data == b"initial_pre_image_bytes"
assert contents[0].parts[2].inline_data.data == b"initial_post_image_bytes"
# 2. Verify index 3 (intermediate image) is replaced by text placeholder
assert getattr(contents[3].parts[1], "inline_data", None) is None
assert "[Screenshot of intermediate step omitted for performance]" in contents[3].parts[1].text
# 3. Verify index 6 (latest image) is fully preserved
assert contents[6].parts[1].inline_data.data == b"latest_live_image_bytes_to_keep"
# The incident carries the EXACT XML error (TARGET_OCCUPIED), NOT the VLM error
incident = result["last_execution_result"]["incident"]
assert incident["kind"] == "safety_net"
assert incident["category"] == ValidationErrorCategory.TARGET_OCCUPIED.value
assert "Occupied by Sign Up button" in incident["reason"]
assert result["open_incident"] == incident
@pytest.mark.asyncio
tests/unit/graph/test_checker_events.py
+348
View File
@@ -0,0 +1,348 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Checker process visibility: every visible step of the Checker (checkpoint
spawn/harvest, final review, run outcome) is published as a ``checker_event``
on the DataEngine bus, and the persisted ledger carries enough context
(subgoal text) for the admin console to rebuild the same view offline."""
import asyncio
import json
import time
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from artemis.agents.checker.checker import CheckReport, CheckVerdict
from artemis.context import ArtemisContext, ExecutionSetup
from artemis.graph.checkpoints import (
CHECKER_EVENT,
CheckpointRun,
PendingCheckpoint,
harvest_run,
publish_checker_event,
read_ledger,
spawn_pending_checkpoints,
)
from artemis.graph.graph import exit_settlement_node
from artemis.graph.state import State
from artemis.utils.plan_grammar import CheckItem, subgoal_hash
GOAL_TEXT = "Create the alarm"
GOAL_KEY = subgoal_hash(GOAL_TEXT)
PLAN = (
f"- [x] {GOAL_TEXT}\n - verify: the alarm list shows 7:30 AM\n - assert: a toast appeared\n"
)
def _make_ctx(tmp_path, **setup_kwargs):
setup_kwargs.setdefault("disable_checker", False)
setup_kwargs.setdefault("disable_midway_checks", False)
ctx = MagicMock(spec=ArtemisContext)
ctx.execution_setup = ExecutionSetup(**setup_kwargs)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = tmp_path
ctx.pending_checkpoints = []
ctx.checkpoint_tasks = {}
ctx.checkpoint_attempt_seq = {}
ctx.checkpoint_repairs = {}
ctx.assert_halt = False
ctx.final_check_attempts = 0
return ctx
def _make_state(**overrides):
state = MagicMock(spec=State)
state.initial_goal = "the user goal"
state.user_stop_requested = False
for k, v in overrides.items():
setattr(state, k, v)
return state
def _write_plan(tmp_path, content=PLAN):
notes = tmp_path / "notes"
notes.mkdir(parents=True, exist_ok=True)
(notes / "task_plan.md").write_text(content, encoding="utf-8")
def _pending():
return PendingCheckpoint(
checkpoint_id=GOAL_KEY,
subgoal_text=GOAL_TEXT,
check_items=(
CheckItem(
kind="verify",
text="the alarm list shows 7:30 AM",
when="on_complete",
parent_key=GOAL_KEY,
),
CheckItem(
kind="assert", text="a toast appeared", when="on_complete", parent_key=GOAL_KEY
),
),
plan_text=PLAN,
trigger_ts=time.time(),
)
def _events(ctx, event: str | None = None) -> list[dict]:
out = []
for call in ctx.data_engine._publish.call_args_list:
if call.args[0] != CHECKER_EVENT:
continue
payload = call.args[1]
if event is None or payload.get("event") == event:
out.append(payload)
return out
def test_publish_is_best_effort_without_engine():
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = None
publish_checker_event(ctx, {"event": "attempt_started"}) # must not raise
ctx.data_engine = MagicMock()
ctx.data_engine._publish.side_effect = RuntimeError("bus down")
publish_checker_event(ctx, {"event": "attempt_started"}) # swallowed
@pytest.mark.asyncio
async def test_spawn_passes_attempt_id_to_checker(tmp_path):
"""The announcement lives inside the checker (it needs the trace id), so
spawn must hand the attempt id over."""
_write_plan(tmp_path)
ctx = _make_ctx(tmp_path)
ctx.pending_checkpoints.append(_pending())
never = asyncio.Event()
seen: dict = {}
async def hang(*a, **k):
seen.update(k)
await never.wait()
with patch("artemis.agents.checker.checker.run_checkpoint_check", side_effect=hang):
await spawn_pending_checkpoints(ctx, _make_state(), anchor_step_id="step-7")
await asyncio.sleep(0)
assert seen.get("attempt_id") == f"{GOAL_KEY}#1"
assert _events(ctx, "attempt_started") == []
for run in ctx.checkpoint_tasks.values():
run.task.cancel()
try:
await run.task
except (asyncio.CancelledError, Exception):
pass
@pytest.mark.asyncio
async def test_checker_entry_announces_attempt_with_trace_id(tmp_path):
"""attempt_started is published from inside the traced checker scope and
carries the Checker's own trace id (UI routing key for its stream/tools),
the anchor and the declared items."""
from artemis.agents.checker.checker import run_checkpoint_check, run_final_check
from artemis.graph.checkpoints import EvidenceAnchor
_write_plan(tmp_path)
ctx = _make_ctx(tmp_path)
ctx.data_engine.get_step_number.return_value = 7
empty_report = CheckReport(verdicts=[])
anchor = EvidenceAnchor(anchor_step_id="step-7", trigger_ts=time.time(), plan_text=PLAN)
with (
patch(
"artemis.agents.checker.checker._run_check_loop", AsyncMock(return_value=empty_report)
),
patch("artemis.agents.checker.checker.build_checker_tools", return_value=[]),
patch("artemis.agents.checker.checker._format_history", return_value=""),
patch(
"artemis.agents.checker.checker._capture_final_screen",
AsyncMock(return_value=(None, "")),
),
):
await run_checkpoint_check(
ctx,
check_items=_pending().check_items,
anchor=anchor,
goal="g",
subgoal_text=GOAL_TEXT,
attempt_id=f"{GOAL_KEY}#1",
)
await run_final_check(
ctx,
goal="g",
plan_text=PLAN,
ledger=[],
check_items=_pending().check_items,
attempt_id="final#1",
)
started = _events(ctx, "attempt_started")
assert [e["phase"] for e in started] == ["checkpoint", "final"]
cp, fin = started
assert cp["attempt_id"] == f"{GOAL_KEY}#1"
assert cp["checkpoint_id"] == GOAL_KEY
assert cp["subgoal_text"] == GOAL_TEXT
assert cp["anchor_step_id"] == "step-7"
assert [i["kind"] for i in cp["items"]] == ["verify", "assert"]
assert cp["trace_id"] # set by the @trace decorator scope
assert "timestamp" in cp and "ts" in cp
assert fin["attempt_id"] == "final#1"
assert fin["checkpoint_id"] == "final"
assert fin["trace_id"]
@pytest.mark.asyncio
async def test_harvest_publishes_verdicts_findings_and_ledger_carries_subgoal(tmp_path):
_write_plan(tmp_path)
ctx = _make_ctx(tmp_path)
report = CheckReport(
verdicts=[
CheckVerdict(
item_text="the alarm list shows 7:30 AM",
kind="verify",
status="failed",
evidence="list shows 7:00 AM",
suggestion="open the alarm and set 7:30",
),
CheckVerdict(
item_text="a toast appeared", kind="assert", status="passed", evidence="toast seen"
),
]
)
task = asyncio.get_event_loop().create_future()
task.set_result(report)
run = CheckpointRun(attempt_id=f"{GOAL_KEY}#1", task=task, checkpoint=_pending())
findings = harvest_run(ctx, _make_state(), run, allow_side_effects=True, anchor_step_id="s1")
finished = _events(ctx, "attempt_finished")
assert len(finished) == 1
ev = finished[0]
assert ev["status"] == "done"
assert ev["phase"] == "checkpoint"
assert ev["subgoal_text"] == GOAL_TEXT
assert [v["status"] for v in ev["verdicts"]] == ["failed", "passed"]
assert ev["verdicts"][0]["suggestion"] == "open the alarm and set 7:30"
assert ev["findings"] == findings and findings # the same strings the Operator sees
assert ev["reverted"] is True
assert ev["applicable"] is True
assert ev["repairs_used"] == 1
# The ledger (offline source for the UI backfill) names the subgoal too.
for rec in read_ledger(tmp_path):
assert rec["subgoal_text"] == GOAL_TEXT
@pytest.mark.asyncio
async def test_harvest_superseded_and_error_publish_terminal_status(tmp_path):
_write_plan(tmp_path)
ctx = _make_ctx(tmp_path)
cancelled = asyncio.get_event_loop().create_future()
cancelled.cancel()
harvest_run(
ctx,
_make_state(),
CheckpointRun(attempt_id=f"{GOAL_KEY}#1", task=cancelled, checkpoint=_pending()),
allow_side_effects=False,
)
errored = asyncio.get_event_loop().create_future()
errored.set_exception(TimeoutError())
harvest_run(
ctx,
_make_state(),
CheckpointRun(attempt_id=f"{GOAL_KEY}#2", task=errored, checkpoint=_pending()),
allow_side_effects=True,
)
finished = _events(ctx, "attempt_finished")
assert [e["status"] for e in finished] == ["superseded", "error"]
assert all(v["status"] == "superseded" for v in finished[0]["verdicts"])
assert all(v["status"] == "inconclusive" for v in finished[1]["verdicts"])
assert "timed out" in finished[1]["error"]
@pytest.mark.asyncio
async def test_final_review_and_run_outcome_are_published(tmp_path):
_write_plan(tmp_path)
ctx = _make_ctx(tmp_path, disable_midway_checks=True)
report = CheckReport(
verdicts=[
CheckVerdict(
item_text="the alarm list shows 7:30 AM",
kind="verify",
status="passed",
evidence="final screen shows 7:30 AM",
),
CheckVerdict(
item_text="a toast appeared",
kind="assert",
status="inconclusive",
evidence="no history evidence",
),
]
)
with patch("artemis.graph.graph.run_final_check", AsyncMock(return_value=report)):
update = await exit_settlement_node(_make_state(), ctx)
assert update["exit_settlement_route"] == "end"
finished = _events(ctx, "attempt_finished")
assert finished and finished[0]["phase"] == "final"
assert finished[0]["attempt_id"] == "final#1"
assert finished[0]["status"] == "done"
assert finished[0]["route"] == "end"
assert [v["status"] for v in finished[0]["verdicts"]] == ["passed", "inconclusive"]
outcome = _events(ctx, "run_outcome")
assert len(outcome) == 1
assert outcome[0]["task_status"] == "completed"
assert outcome[0]["tests"]["passed"] == 1
assert outcome[0]["tests"]["inconclusive"] == 1
assert outcome[0]["tests"]["failed"] == 0
assert outcome[0]["last_findings"] == []
# Persisted mirror of the same event for the UI backfill endpoint.
persisted = json.loads((tmp_path / "run_outcome.json").read_text(encoding="utf-8"))
assert persisted["task_status"] == "completed"
assert all(r["subgoal_text"] for r in read_ledger(tmp_path))
@pytest.mark.asyncio
async def test_final_review_bounce_back_publishes_continue_route(tmp_path):
_write_plan(tmp_path)
ctx = _make_ctx(tmp_path, disable_midway_checks=True, final_check_max_attempts=3)
report = CheckReport(
verdicts=[
CheckVerdict(
item_text="the alarm list shows 7:30 AM",
kind="verify",
status="failed",
evidence="list is empty",
suggestion="create it",
)
],
unmet_subgoals=[GOAL_TEXT],
)
with patch("artemis.graph.graph.run_final_check", AsyncMock(return_value=report)):
update = await exit_settlement_node(_make_state(), ctx)
assert update["exit_settlement_route"] == "continue"
finished = _events(ctx, "attempt_finished")
assert finished[0]["route"] == "continue"
assert finished[0]["unmet_subgoals"] == [GOAL_TEXT]
assert finished[0]["findings"] == update["operator_feedback"]
# No run outcome while the loop continues.
assert _events(ctx, "run_outcome") == []
tests/unit/graph/test_checkpoints.py
+32 -22
View File
@@ -68,7 +68,6 @@ def _make_ctx(tmp_path, **setup_kwargs):
ctx.planner_task = None
ctx.last_validated_plan = None
ctx.pending_validated_plan = None
ctx.task_plan_content_before = None
return ctx
@@ -232,7 +231,7 @@ async def test_execution_check_spawns_after_record_step_with_correct_anchor(tmp_
captured = {}
async def fake_check(ctx_arg, check_items, anchor, goal, subgoal_text):
async def fake_check(ctx_arg, check_items, anchor, goal, subgoal_text, attempt_id=None):
captured["anchor"] = anchor
captured["goal"] = goal
return CheckReport(verdicts=[])
@@ -273,24 +272,41 @@ async def test_execution_check_no_spawn_after_user_stop(tmp_path):
@pytest.mark.asyncio
async def test_record_step_on_planner_rejected_turn(tmp_path):
"""Every operator turn leaves a step record — the planner-rejected turn is
recorded too, tagged planner_rejected."""
_write_plan(tmp_path)
async def test_planner_flag_is_advisory_hint_only(tmp_path):
"""A flagged plan change is advisory: no rollback, the turn's actions
still run, the Operator gets the concern + reason as feedback, the turn
is recorded (tagged planner_flagged) and the ratchet baseline advances so
the same change is not re-flagged on every later write."""
plan_path = _write_plan(tmp_path)
plan_on_disk = plan_path.read_text(encoding="utf-8")
ctx = _make_ctx(tmp_path)
ctx.data_engine.record_step.return_value = "rejected-step"
rejection = asyncio.get_event_loop().create_future()
rejection.set_result({"status": "failed", "feedback": "bad plan"})
ctx.planner_task = rejection
ctx.data_engine.record_step.return_value = "flagged-step"
ctx.last_validated_plan = "- [ ] old baseline\n"
ctx.pending_validated_plan = plan_on_disk
flagged = asyncio.get_event_loop().create_future()
flagged.set_result({"status": "failed", "feedback": "milestone 2 drifts from the goal"})
ctx.planner_task = flagged
state = _make_state(operator_raw_data=_raw_data())
with patch("artemis.graph.graph._get_active_subgoal_hashes", return_value=("h", None)):
update = await execution_check_node(state, ctx)
assert update["checker_success"] is False
assert update["current_step_id"] == "rejected-step"
# Actions are not suppressed and the plan file is untouched.
assert update["checker_success"] is True
assert "structured_decisions" not in update
assert plan_path.read_text(encoding="utf-8") == plan_on_disk
assert update["current_step_id"] == "flagged-step"
extra = ctx.data_engine.record_step.call_args.kwargs["extra_metadata"]
assert extra.get("planner_rejected") is True
assert extra.get("planner_flagged") is True
# Hint + reason reach the Operator through operator_feedback.
feedback = update["operator_feedback"]
assert feedback and feedback[0].startswith("[planner]")
assert "milestone 2 drifts from the goal" in feedback[0]
assert "NOT rolled back" in feedback[0]
# Baseline advanced to the reviewed content.
assert ctx.last_validated_plan == plan_on_disk
assert ctx.pending_validated_plan is None
assert ctx.planner_task is None
@pytest.mark.asyncio
@@ -584,9 +600,7 @@ async def test_finding_line_regrows_on_model_plan_write(tmp_path):
with patch("artemis.graph.graph.invoke_tool_with_injection", side_effect=fake_invoke):
wrapped = wrap_note_tool(ctx, mock_tool)
await wrapped.ainvoke(
{"key": "task_plan", "content": clean_rewrite, "tool_call_id": "t1"}
)
await wrapped.ainvoke({"key": "task_plan", "content": clean_rewrite, "tool_call_id": "t1"})
# ...and the deterministic projection grows it back on the write path.
assert "- finding: verify failed" in plan_path.read_text(encoding="utf-8")
@@ -625,9 +639,7 @@ async def test_finding_line_removed_on_repair_quota_exhaustion(tmp_path):
# Re-complete, fail again: quota (1) is exhausted -> finding settled.
plan_path.write_text(
plan_path.read_text(encoding="utf-8").replace(
f"- [/] {GOAL_TEXT}", f"- [x] {GOAL_TEXT}"
),
plan_path.read_text(encoding="utf-8").replace(f"- [/] {GOAL_TEXT}", f"- [x] {GOAL_TEXT}"),
encoding="utf-8",
)
run2 = _done_run(
@@ -652,9 +664,7 @@ async def test_checker_note_prefix_writes_rejected_by_wrappers(tmp_path):
update_tool.name = "update_note"
update_tool.description = "update"
with patch(
"artemis.graph.graph.invoke_tool_with_injection", new_callable=AsyncMock
) as inv:
with patch("artemis.graph.graph.invoke_tool_with_injection", new_callable=AsyncMock) as inv:
wrapped_save = wrap_note_tool(ctx, save_tool)
result = await wrapped_save.ainvoke(
{"key": checker_note_key(GOAL_KEY), "content": "spoof", "tool_call_id": "t1"}
tests/unit/graph/test_exit_settlement.py
+22 -1
View File
@@ -143,6 +143,27 @@ def test_gate_incomplete_plan_continues(tmp_path):
assert convergence_gate(_make_state(), ctx) == "continue"
def test_gate_unevaluated_plan_validation_is_not_a_failure(tmp_path):
"""First pass after the Planner node: ``checker_success`` is still None
(never evaluated). That must not short-circuit the gate — a fully done
plan still routes to the exit, and no "validation failed" log fires."""
_write_plan(tmp_path, DONE_PLAN_NO_CHECKS)
ctx = _make_ctx(tmp_path)
with patch("artemis.graph.graph.logger") as mock_logger:
assert convergence_gate(_make_state(checker_success=None), ctx) == "exit_settlement"
logged = " ".join(str(c.args[0]) for c in mock_logger.info.call_args_list if c.args)
assert "Plan validation failed" not in logged
def test_gate_explicit_validation_failure_still_continues(tmp_path):
_write_plan(tmp_path, DONE_PLAN_NO_CHECKS)
ctx = _make_ctx(tmp_path)
with patch("artemis.graph.graph.logger") as mock_logger:
assert convergence_gate(_make_state(checker_success=False), ctx) == "continue"
logged = " ".join(str(c.args[0]) for c in mock_logger.info.call_args_list if c.args)
assert "Plan validation failed" in logged
def test_gate_assert_halt_terminates_mid_plan(tmp_path):
_write_plan(tmp_path, "- [/] Create alarm\n - assert: toast appeared\n")
ctx = _make_ctx(tmp_path)
@@ -217,7 +238,7 @@ async def test_final_check_receives_ledger_and_at_end_items(tmp_path):
captured = {}
async def fake_final(ctx_arg, goal, plan_text, ledger, check_items):
async def fake_final(ctx_arg, goal, plan_text, ledger, check_items, attempt_id=None):
captured["goal"] = goal
captured["ledger"] = ledger
captured["items"] = list(check_items)
tests/unit/graph/test_graph.py
-3
View File
@@ -39,7 +39,6 @@ def _make_ctx(base_dir, *, disable_checker=True, disable_planner_validation=Fals
ctx.pending_checkpoints = []
ctx.checkpoint_tasks = {}
ctx.pending_validated_plan = None
ctx.task_plan_content_before = None
return ctx
@@ -192,8 +191,6 @@ async def test_ratchet_baseline_survives_consecutive_small_edits(tmp_path):
# Both validation runs were anchored to the original baseline
for call in mock_validation.call_args_list:
assert call.args[2] == ORIGINAL_PLAN
# Rollback target preserves the last baseline-consistent content
assert ctx.task_plan_content_before == ORIGINAL_PLAN
@pytest.mark.asyncio
tests/unit/mcp/test_action_executor_helper_tools.py
+283
View File
@@ -0,0 +1,283 @@
"""Helper-tool routing in McpActionExecutor (Flash profile).
The Flash runner binds ``ask_explorer``, ``video_analyzer`` and
``recall_history`` declarations next to its device actions; the executor must
route them to the same implementations the Pro operator's LangChain tools
delegate to, as agent-side text tools (no screenshot capture, no device
action), and report their status explicitly instead of sniffing the text for
the word "Error".
"""
from unittest.mock import AsyncMock, Mock, patch
import pytest
from artemis.mcp.action_executor import AGENT_TOOL_NAMES, McpActionExecutor
from artemis.tools.explorer_tool import ExplorerCandidate, ExplorerOutcome
def _make_executor(**kwargs):
ctx = Mock()
ctx.data_engine = None
ctx.device.device_width = 1080
ctx.device.device_height = 2400
actuator = Mock()
actuator.controller = Mock()
actuator.extensions.return_value = []
return McpActionExecutor(ctx, actuator=actuator, **kwargs)
def _state():
state = Mock()
state.latest_screenshot = "/tmp/shot.jpg"
state.indexed_points = []
state.indexed_elements = []
state.operator_raw_data = None
return state
def test_helper_tools_are_not_device_actions():
executor = _make_executor()
assert {"ask_explorer", "video_analyzer", "recall_history"} <= AGENT_TOOL_NAMES
assert not (AGENT_TOOL_NAMES & executor.action_tool_names)
# ---------------------------------------------------------------------------
# ask_explorer
# ---------------------------------------------------------------------------
def test_executor_agent_name_defaults_to_validator_and_is_configurable():
assert _make_executor().agent_name == "validator"
assert _make_executor(agent_name="flash").agent_name == "flash"
@pytest.mark.asyncio
async def test_ask_explorer_runs_the_pipeline_for_the_executor_agent():
"""The tier is resolved from the executor's agent name, never from the call."""
executor = _make_executor(agent_name="flash")
state = _state()
outcome = ExplorerOutcome(
candidates=[ExplorerCandidate(label="S1", coords=(500, 500), description="Send")]
)
with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=outcome)) as locate:
result = await executor.execute(
"ask_explorer", {"query": "blue Send button", "context_feedback": "n/a"}, "tc", state
)
locate.assert_awaited_once_with(
executor.ctx, state, "blue Send button", "n/a", agent_name="flash"
)
assert result.status == "success"
assert "Explorer located 1 candidate(s) for 'blue Send button'" in result.text_summary
assert "[1] 'Send' at normalized [500, 500]" in result.text_summary
# Candidates were registered on the state for index-based actions.
assert state.indexed_points == [[540, 1200]]
assert state.indexed_elements[0]["index"] == 1
assert result.screenshot_bytes is None
assert result.ui_elements_text is None
@pytest.mark.asyncio
async def test_ask_explorer_clean_not_found_is_a_successful_answer():
executor = _make_executor()
outcome = ExplorerOutcome(message="Nothing like that is visible; the keyboard covers it.")
with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=outcome)):
result = await executor.execute("ask_explorer", {"query": "gear icon"}, "tc", _state())
assert result.status == "success"
assert "Explorer could not locate 'gear icon'" in result.text_summary
assert "keyboard covers it" in result.text_summary
@pytest.mark.asyncio
async def test_ask_explorer_run_failure_is_an_error_result():
executor = _make_executor()
outcome = ExplorerOutcome.failure("Explorer failed: model quota exhausted")
with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=outcome)):
result = await executor.execute("ask_explorer", {"query": "gear icon"}, "tc", _state())
assert result.status == "error"
assert "Explorer could not run for 'gear icon'" in result.text_summary
assert "model quota exhausted" in result.text_summary
@pytest.mark.asyncio
async def test_ask_explorer_accepts_the_legacy_task_description_alias():
executor = _make_executor()
outcome = ExplorerOutcome(message="not here")
with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=outcome)) as locate:
result = await executor.execute(
"ask_explorer", {"task_description": "gear icon top-right"}, "tc", _state()
)
assert locate.await_args.args[2] == "gear icon top-right"
assert locate.await_args.args[3] == ""
assert result.status == "success"
@pytest.mark.asyncio
async def test_ask_explorer_exception_is_contained():
executor = _make_executor()
with patch(
"artemis.tools.explorer_tool.locate", new=AsyncMock(side_effect=RuntimeError("boom"))
):
result = await executor.execute("ask_explorer", {"query": "x"}, "tc", _state())
assert result.status == "error"
assert result.text_summary == "Error executing ask_explorer: boom"
# ---------------------------------------------------------------------------
# Legacy MobileActionExecutor parity (same ask_explorer semantics)
# ---------------------------------------------------------------------------
def _make_legacy_executor(**kwargs):
from artemis.agents.validator.tool_declarations import MobileActionExecutor
ctx = Mock()
ctx.data_engine = None
ctx.device.device_width = 1080
ctx.device.device_height = 2400
actuator = Mock()
actuator.controller = Mock()
return MobileActionExecutor(ctx, actuator=actuator, **kwargs)
@pytest.mark.asyncio
async def test_legacy_executor_ask_explorer_status_follows_the_outcome():
executor = _make_legacy_executor()
assert executor.agent_name == "validator"
not_found = ExplorerOutcome(message="Not visible.")
with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=not_found)):
# The pipeline functions are imported at module load in the legacy executor.
with patch(
"artemis.agents.validator.tool_declarations.locate",
new=AsyncMock(return_value=not_found),
) as locate:
result = await executor.execute(
"ask_explorer", {"task_description": "gear icon"}, "tc", _state()
)
assert locate.await_args.args[2] == "gear icon"
assert locate.await_args.kwargs["agent_name"] == "validator"
assert result.status == "success" # a clean not-found is an answer, not a failure
failed = ExplorerOutcome.failure("Explorer failed: boom")
with patch(
"artemis.agents.validator.tool_declarations.locate", new=AsyncMock(return_value=failed)
):
result = await executor.execute("ask_explorer", {"query": "gear icon"}, "tc", _state())
assert result.status == "error"
@pytest.mark.asyncio
async def test_legacy_executor_agent_name_reaches_the_tier_resolver():
executor = _make_legacy_executor(agent_name="flash")
with patch(
"artemis.agents.validator.tool_declarations.locate",
new=AsyncMock(return_value=ExplorerOutcome(message="no")),
) as locate:
await executor.execute("ask_explorer", {"query": "x"}, "tc", _state())
assert locate.await_args.kwargs["agent_name"] == "flash"
# ---------------------------------------------------------------------------
# video_analyzer
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_video_analyzer_routes_to_the_subagent_as_a_text_tool():
executor = _make_executor()
with patch("artemis.agents.video_analyzer.video_analyzer.VideoAnalyzer") as analyzer_cls:
analyzer_cls.return_value.run = AsyncMock(
return_value=("The ad finished at 12s; an Error dialog never appeared.", "success")
)
result = await executor.execute(
"video_analyzer",
{"time_description": "from 5s to 12s", "purpose": "what played"},
"tc-video",
None,
)
analyzer_cls.return_value.run.assert_awaited_once_with("from 5s to 12s", "what played")
assert result.status == "success" # explicit status, not text sniffing
assert result.text_summary.startswith("The ad finished at 12s")
assert result.screenshot_bytes is None
assert result.ui_elements_text is None
assert result.raw_result is None
@pytest.mark.asyncio
async def test_video_analyzer_failure_is_an_error_result():
executor = _make_executor()
with patch("artemis.agents.video_analyzer.video_analyzer.VideoAnalyzer") as analyzer_cls:
analyzer_cls.return_value.run = AsyncMock(return_value=("no recording found", "failed"))
result = await executor.execute(
"video_analyzer", {"time_description": "from 0s to 3s", "purpose": "x"}, "tc", None
)
assert result.status == "error"
assert result.text_summary == "Video analysis failed: no recording found"
@pytest.mark.asyncio
async def test_video_analyzer_exception_is_contained():
executor = _make_executor()
with patch("artemis.agents.video_analyzer.video_analyzer.VideoAnalyzer") as analyzer_cls:
analyzer_cls.return_value.run = AsyncMock(side_effect=RuntimeError("boom"))
result = await executor.execute(
"video_analyzer", {"time_description": "from 0s to 3s", "purpose": "x"}, "tc", None
)
assert result.status == "error"
assert "boom" in result.text_summary
# ---------------------------------------------------------------------------
# recall_history
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_recall_history_routes_to_the_shared_tool_with_text_result():
executor = _make_executor()
with patch("artemis.tools.history_recall.recall_history") as tool:
tool.execute = AsyncMock(return_value="- Step 3 (T+00:30): tap btn3 -> executed")
result = await executor.execute(
"recall_history",
{"query": "login Error timeout", "step_range": [1, 5], "unknown_arg": 1},
"tc-recall",
None,
)
tool.execute.assert_awaited_once_with(
ctx=executor.ctx, query="login Error timeout", step_range=[1, 5]
)
# A lookup answer mentioning "Error" is still a successful lookup.
assert result.status == "success"
assert result.text_summary == "- Step 3 (T+00:30): tap btn3 -> executed"
assert result.raw_result is None
@pytest.mark.asyncio
async def test_recall_history_forwards_multimodal_blocks():
executor = _make_executor()
blocks = [
{"type": "text", "text": "Step 4 matched."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,AAA"}},
]
with patch("artemis.tools.history_recall.recall_history") as tool:
tool.execute = AsyncMock(return_value=blocks)
result = await executor.execute(
"recall_history", {"query": "settings", "include_images": True}, "tc", None
)
assert result.status == "success"
assert result.text_summary == "Step 4 matched."
assert result.raw_result == blocks
@pytest.mark.asyncio
async def test_recall_history_exception_is_contained():
executor = _make_executor()
with patch("artemis.tools.history_recall.recall_history") as tool:
tool.execute = AsyncMock(side_effect=RuntimeError("db gone"))
result = await executor.execute("recall_history", {"query": "x"}, "tc", None)
assert result.status == "success" # contained: the answer explains the failure
assert result.text_summary == "recall_history failed: db gone"
tests/unit/mcp/test_action_manifest.py
+2 -5
View File
@@ -43,14 +43,11 @@ from artemis.mcp.action_manifest import (
REPO_ROOT = Path(__file__).resolve().parents[3]
# Prompt sources belonging to the three agents this manifest governs (plus the
# Prompt sources belonging to the agents this manifest governs (plus the
# Validator's safety net). Other agents (log_analyzer, image_processor, ...) have their
# own tool universes and are deliberately out of scope.
SCOPED_PROMPT_FILES = [
"artemis/agents/flash/flash_runner.md",
"artemis/agents/validator/failure_analyzer.md",
"artemis/agents/validator/target_disappeared_analyzer.md",
"artemis/agents/validator/pixel_target_disappeared_analyzer.md",
"artemis/agents/validator/pixel_safety_net.md",
]
@@ -284,4 +281,4 @@ def test_filter_rejects_unknown_agent_role():
def test_all_agent_roles_are_covered():
assert AGENT_ROLES == {"operator", "flash", "failure_analyzer", "validator"}
assert AGENT_ROLES == {"operator", "flash", "validator"}
tests/unit/mcp/test_action_specs.py
-1
View File
@@ -127,7 +127,6 @@ def test_validator_declaration_order_is_stable():
"list_notes",
"manage_app",
"wait_for_delay",
"report_failure_analysis",
]
tests/unit/mcp/test_actuators.py
+1 -3
View File
@@ -64,9 +64,7 @@ async def test_click_sequence_taps_each_point(actuator):
async def test_long_press_message_and_duration(actuator):
res = await actuator.long_press(250, 750, duration_ms=1500)
assert res.ok
assert res.message == (
"Long pressed at [250, 750] (normalized) for 1500ms successfully."
)
assert res.message == ("Long pressed at [250, 750] (normalized) for 1500ms successfully.")
assert res.duration_ms == 1500
tests/unit/mcp/test_adb_server_contract.py
+1 -3
View File
@@ -121,9 +121,7 @@ def _prop(tool, name: str) -> dict:
@pytest.mark.asyncio
async def test_adb_server_manifest_matches_fixture():
expected = json.loads(
(FIXTURES / "adb_server_manifest.json").read_text(encoding="utf-8")
)
expected = json.loads((FIXTURES / "adb_server_manifest.json").read_text(encoding="utf-8"))
tools = await _adb_server_tools()
generated = {
name: {"description": t.description, "inputSchema": t.inputSchema}
tests/unit/mcp/test_background_task_runner.py
+67
View File
@@ -35,3 +35,70 @@ async def test_background_agent_initialization_forwards_health_settings():
)
agent.init.assert_awaited_once_with(retry_count=3, retry_wait_seconds=4)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("knobs", "expect_level", "expect_mode"),
[
({"verification_level": "strict", "explorer_pro_mode": "ultra"}, "strict", "ultra"),
({}, None, None),
],
)
async def test_run_task_applies_pro_tuning_to_agent_config(
tmp_path, monkeypatch, knobs, expect_level, expect_mode
):
"""The detached runner applies --verification-level / --explorer-pro-mode on the builder."""
from types import SimpleNamespace
from unittest.mock import patch
from artemis.runtime import trace_store
from mcp_server.background import task_runner as bg
monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path))
trace_id = "trace-tuning"
trace_store.init_trace(
trace_id=trace_id,
task_desc="Audit checkout",
model="Pro",
conversation_id="",
device_serial="emulator-5554",
)
fake_builder = MagicMock()
fake_builders = MagicMock()
fake_builders.AgentConfig.with_default_profile.return_value = fake_builder
fake_agent = MagicMock()
fake_agent._device_context = SimpleNamespace(device_id="emulator-5554")
fake_agent.run_task = AsyncMock(return_value="done")
fake_agent.clean = AsyncMock()
monkeypatch.setattr(bg.device_utils, "resolve_adb_path", lambda: "adb")
monkeypatch.setattr(bg.device_utils, "get_connected_devices", lambda _adb: ["emulator-5554"])
monkeypatch.setattr(bg, "resolve_profile_file", lambda: None)
monkeypatch.setattr(bg, "_initialize_agent", AsyncMock())
monkeypatch.setattr(bg, "notify", MagicMock())
with (
patch("artemis.sdk.builders.Builders", fake_builders),
patch("artemis.sdk.Agent", return_value=fake_agent),
patch("artemis.sdk.types.AgentProfile", MagicMock()),
patch("artemis.config.initialize_llm_config", return_value=MagicMock()),
):
await bg.run_task(
trace_id=trace_id,
task_desc="Audit checkout",
model="Pro",
conversation_id="",
device_serial="emulator-5554",
**knobs,
)
if expect_level is None:
fake_builder.with_verification_level.assert_not_called()
fake_builder.with_explorer.assert_not_called()
else:
fake_builder.with_verification_level.assert_called_once_with(expect_level)
fake_builder.with_explorer.assert_called_once_with(pro_mode=expect_mode)
fake_agent.run_task.assert_awaited_once()
assert trace_store.read_status(trace_id)["status"] == "completed"
tests/unit/mcp/test_device_utils.py
+50
View File
@@ -0,0 +1,50 @@
"""Unit tests for platform-specific emulator process startup."""
from types import SimpleNamespace
from unittest.mock import MagicMock
from mcp_server.utils import device_utils
def test_ensure_emulator_uses_windows_creation_flags(monkeypatch) -> None:
popen = MagicMock()
monkeypatch.setattr(device_utils.sys, "platform", "win32")
monkeypatch.setattr(device_utils, "is_emulator_running", lambda _adb: False)
monkeypatch.setattr(device_utils.os.path, "exists", lambda _path: True)
monkeypatch.setattr(device_utils.subprocess, "Popen", popen)
monkeypatch.setattr(
device_utils.subprocess,
"run",
lambda *_args, **_kwargs: SimpleNamespace(stdout="1"),
)
monkeypatch.setattr(device_utils.time, "sleep", lambda _seconds: None)
assert device_utils.ensure_emulator(
adb_path="adb.exe", emulator_path="emulator.exe", timeout_seconds=1
)
kwargs = popen.call_args.kwargs
assert kwargs["creationflags"] == (
device_utils.subprocess.CREATE_NEW_PROCESS_GROUP | device_utils.subprocess.DETACHED_PROCESS
)
assert "start_new_session" not in kwargs
def test_ensure_emulator_starts_new_session_on_posix(monkeypatch) -> None:
popen = MagicMock()
monkeypatch.setattr(device_utils.sys, "platform", "linux")
monkeypatch.setattr(device_utils, "is_emulator_running", lambda _adb: False)
monkeypatch.setattr(device_utils.os.path, "exists", lambda _path: True)
monkeypatch.setattr(device_utils.subprocess, "Popen", popen)
monkeypatch.setattr(
device_utils.subprocess,
"run",
lambda *_args, **_kwargs: SimpleNamespace(stdout="1"),
)
monkeypatch.setattr(device_utils.time, "sleep", lambda _seconds: None)
assert device_utils.ensure_emulator(adb_path="adb", emulator_path="emulator", timeout_seconds=1)
kwargs = popen.call_args.kwargs
assert kwargs["start_new_session"] is True
assert "creationflags" not in kwargs
tests/unit/mcp/test_mcp_stdio_lifecycle.py
+18 -6
View File
@@ -66,7 +66,9 @@ def test_mcp_stdio_handshake_immediate_input():
without hanging, even when stdin data is pushed concurrently with process startup.
"""
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
project_root = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
)
python_exe = sys.executable
p = subprocess.Popen(
@@ -118,7 +120,9 @@ def test_mcp_stdio_handshake_immediate_input():
# Read initialize response with a strict timeout
init_resp_line = _readline_with_timeout(p.stdout, 6.0)
assert init_resp_line is not None, "MCP server failed to respond to initialize request within 6 seconds (deadlock detected)!"
assert init_resp_line is not None, (
"MCP server failed to respond to initialize request within 6 seconds (deadlock detected)!"
)
init_data = json.loads(init_resp_line)
assert init_data.get("id") == 1
assert "result" in init_data
@@ -160,7 +164,9 @@ def test_awake_service_adb_command_isolates_stdin():
assert mock_run.called
kwargs = mock_run.call_args.kwargs
assert kwargs.get("stdin") == subprocess.DEVNULL, "Expected stdin=subprocess.DEVNULL to prevent stdin hijacking!"
assert kwargs.get("stdin") == subprocess.DEVNULL, (
"Expected stdin=subprocess.DEVNULL to prevent stdin hijacking!"
)
def test_awake_lease_run_isolates_stdin():
@@ -172,19 +178,25 @@ def test_awake_lease_run_isolates_stdin():
assert mock_run.called
kwargs = mock_run.call_args.kwargs
assert kwargs.get("stdin") == subprocess.DEVNULL, "Expected stdin=subprocess.DEVNULL to prevent stdin hijacking!"
assert kwargs.get("stdin") == subprocess.DEVNULL, (
"Expected stdin=subprocess.DEVNULL to prevent stdin hijacking!"
)
def test_detached_process_kwargs_isolates_stdin():
"""Verify get_detached_process_kwargs always sets stdin=subprocess.DEVNULL."""
kwargs = env_utils.get_detached_process_kwargs()
assert kwargs.get("stdin") == subprocess.DEVNULL, "Expected stdin=subprocess.DEVNULL for detached background tasks!"
assert kwargs.get("stdin") == subprocess.DEVNULL, (
"Expected stdin=subprocess.DEVNULL for detached background tasks!"
)
def test_device_utils_isolates_stdin():
"""Verify device_utils subprocess calls always set stdin=subprocess.DEVNULL."""
with patch("mcp_server.utils.device_utils.subprocess.run") as mock_run:
mock_run.return_value = MagicMock(returncode=0, stdout="List of devices attached\n", stderr="")
mock_run.return_value = MagicMock(
returncode=0, stdout="List of devices attached\n", stderr=""
)
device_utils.get_connected_devices()
assert mock_run.called
assert mock_run.call_args.kwargs.get("stdin") == subprocess.DEVNULL
tests/unit/mcp/test_mcp_tools.py
+100 -6
View File
@@ -53,6 +53,13 @@ def test_tool_signatures():
assert "app_path" in sig_run.parameters
assert "expected_output_desc" in sig_run.parameters
assert "device_serial" in sig_run.parameters
assert "verification_level" in sig_run.parameters
assert "explorer_mode" in sig_run.parameters
assert sig_run.parameters["verification_level"].default is None
assert sig_run.parameters["explorer_mode"].default is None
# The tool description is the only schema an MCP caller sees.
assert "verification_level" in (mobile_run_task.__doc__ or "")
assert "explorer_mode" in (mobile_run_task.__doc__ or "")
# mobile_manage_task signature check
sig_manage = inspect.signature(mobile_manage_task)
@@ -185,6 +192,83 @@ def test_mobile_run_task_dispatched_to_daemon(temp_trace_env, monkeypatch):
assert "stderr_log" in result
@pytest.mark.parametrize(
("kwargs", "match"),
[
({"verification_level": "paranoid"}, "Invalid verification_level"),
({"explorer_mode": "turbo"}, "Invalid explorer_mode"),
],
)
def test_mobile_run_task_rejects_unknown_pro_tuning_before_creating_a_trace(
temp_trace_env, kwargs, match
):
with pytest.raises(ValueError, match=match):
mobile_run_task(task_desc="Open Settings", model="Pro", **kwargs)
# Rejected before init_trace: nothing was written to the trace store.
assert os.listdir(temp_trace_env) == []
def test_mobile_run_task_forwards_pro_tuning_to_background_runner(temp_trace_env):
process = MagicMock(pid=777)
with (
patch("mcp_server.tools.task_runner.DeviceExecutionLock.reserve", return_value="t"),
patch("mcp_server.tools.task_runner.DeviceExecutionLock.transfer_reservation"),
patch("mcp_server.tools.task_runner.subprocess.Popen", return_value=process) as popen,
):
mobile_run_task(
task_desc="Audit the checkout flow",
model="Pro",
verification_level=" Strict ",
explorer_mode="ULTRA",
)
cmd = popen.call_args.args[0]
# Same flag spelling as `artemis run` / the admin-console worker, normalised values.
assert cmd[cmd.index("--verification-level") + 1] == "strict"
assert cmd[cmd.index("--explorer-pro-mode") + 1] == "ultra"
def test_mobile_run_task_omits_pro_tuning_flags_when_unset(temp_trace_env):
process = MagicMock(pid=778)
with (
patch("mcp_server.tools.task_runner.DeviceExecutionLock.reserve", return_value="t"),
patch("mcp_server.tools.task_runner.DeviceExecutionLock.transfer_reservation"),
patch("mcp_server.tools.task_runner.subprocess.Popen", return_value=process) as popen,
):
mobile_run_task(task_desc="Open Settings", model="Pro", verification_level=" ")
cmd = popen.call_args.args[0]
assert "--verification-level" not in cmd
assert "--explorer-pro-mode" not in cmd
def test_mobile_run_task_forwards_pro_tuning_to_daemon(temp_trace_env, monkeypatch):
monkeypatch.delenv("ARTEMIS_STANDALONE", raising=False)
with (
patch(
"mcp_server.tools.task_runner.ensure_daemon_running",
return_value=(True, "http://127.0.0.1:8000"),
),
patch(
"mcp_server.tools.task_runner.submit_task_to_daemon",
return_value={"status": "started", "tasks": [{"session_id": "daemon-sid-2"}]},
) as submit,
patch("mcp_server.tools.task_runner.subprocess.Popen") as popen,
):
result = mobile_run_task(
task_desc="Audit via Daemon",
model="Pro",
verification_level="Checkpoints",
explorer_mode="pro",
)
popen.assert_not_called()
assert result["trace_id"] == "daemon-sid-2"
# JSON field spelling of /api/run: verification_level / explorer_mode.
assert submit.call_args.kwargs["verification_level"] == "checkpoints"
assert submit.call_args.kwargs["explorer_mode"] == "pro"
def test_mobile_manage_task_unknown_trace(temp_trace_env):
res = mobile_manage_task(action="status", trace_id="non-existent-trace")
assert res["status"] == "unknown"
@@ -304,7 +388,9 @@ async def test_mobile_get_device_state_passes_device_serial():
)
with (
patch("mcp_server.tools.device_state._get_controller", return_value=controller) as get_ctrl_mock,
patch(
"mcp_server.tools.device_state._get_controller", return_value=controller
) as get_ctrl_mock,
patch("mcp_server.tools.device_state.is_ocr_configured", return_value=False),
):
await mobile_get_device_state(view_type="hierarchy", device_serial="device-serial-abc")
@@ -317,14 +403,22 @@ async def test_mobile_inspect_trace_includes_device_serial(temp_trace_env):
import sqlite3
trace_id = str(uuid.uuid4())
trace_store.init_trace(trace_id, "Inspect test", "Flash", "conv-1", device_serial="pixel-target-999")
trace_store.init_trace(
trace_id, "Inspect test", "Flash", "conv-1", device_serial="pixel-target-999"
)
# Create dummy database and table
db_path = os.path.join(temp_trace_env, "data_engine.db")
conn = sqlite3.connect(db_path)
conn.execute("CREATE TABLE sessions (session_id TEXT PRIMARY KEY, start_time REAL, device_info TEXT)")
conn.execute("CREATE TABLE steps (step_id TEXT PRIMARY KEY, session_id TEXT, step_number INTEGER, timestamp REAL, pre_image_name TEXT, post_image_name TEXT, summary TEXT, action_taken TEXT, operator_raw_thinking TEXT, last_execution_result TEXT, extra_metadata TEXT)")
conn.execute("CREATE TABLE traces (trace_id TEXT PRIMARY KEY, step_id TEXT, session_id TEXT, type TEXT, name TEXT, status TEXT, timestamp REAL, duration REAL, payload TEXT)")
conn.execute(
"CREATE TABLE sessions (session_id TEXT PRIMARY KEY, start_time REAL, device_info TEXT)"
)
conn.execute(
"CREATE TABLE steps (step_id TEXT PRIMARY KEY, session_id TEXT, step_number INTEGER, timestamp REAL, pre_image_name TEXT, post_image_name TEXT, summary TEXT, action_taken TEXT, operator_raw_thinking TEXT, last_execution_result TEXT, extra_metadata TEXT)"
)
conn.execute(
"CREATE TABLE traces (trace_id TEXT PRIMARY KEY, step_id TEXT, session_id TEXT, type TEXT, name TEXT, status TEXT, timestamp REAL, duration REAL, payload TEXT)"
)
conn.execute("INSERT INTO sessions VALUES (?, ?, ?)", (trace_id, 1000.0, None))
conn.commit()
conn.close()
@@ -362,6 +456,7 @@ async def test_mcp_server_auto_registers_tools():
def test_mobile_manage_task_syncs_terminal_status_from_db(temp_trace_env):
import sqlite3
trace_id = str(uuid.uuid4())
trace_store.init_trace(trace_id, "Running task to sync", "Flash", "conv-sync")
@@ -406,4 +501,3 @@ def test_mobile_manage_task_stop_via_daemon(temp_trace_env, monkeypatch):
mock_stop.assert_called_once_with(trace_id)
assert res["status"] == "cancelled"
assert trace_store.read_status(trace_id)["status"] == "cancelled"
tests/unit/mcp/test_notifiers.py
+6 -3
View File
@@ -107,9 +107,13 @@ def test_desktop_notifier_enabled_by_default(monkeypatch):
import sys
if sys.platform == "linux":
monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/notify-send" if cmd == "notify-send" else None)
monkeypatch.setattr(
shutil, "which", lambda cmd: "/usr/bin/notify-send" if cmd == "notify-send" else None
)
elif sys.platform == "darwin":
monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/osascript" if cmd == "osascript" else None)
monkeypatch.setattr(
shutil, "which", lambda cmd: "/usr/bin/osascript" if cmd == "osascript" else None
)
notifier = DesktopNotifier()
assert notifier.is_available() is True
@@ -228,4 +232,3 @@ def test_agentapi_notifier_candidate_recovery_and_retry(monkeypatch, tmp_path):
assert saved_sessions == [("localhost:1234", "good-token")]
assert os.environ["ANTIGRAVITY_LS_ADDRESS"] == "localhost:1234"
assert os.environ["ANTIGRAVITY_CSRF_TOKEN"] == "good-token"
tests/unit/mcp/test_spawn_watchdog.py
+24 -8
View File
@@ -25,8 +25,12 @@ def test_watchdog_passes_when_runner_creates_logs(tmp_path):
with _patch_trace_dir(tmp_path), patch.object(spawn_tool, "_kill_process_tree") as kill:
assert spawn_tool._watch_spawn(
trace_id, pid=999999, queue_ticket="t", conversation_id=None,
deadline_seconds=0.2, poll_interval=0.05,
trace_id,
pid=999999,
queue_ticket="t",
conversation_id=None,
deadline_seconds=0.2,
poll_interval=0.05,
)
kill.assert_not_called()
@@ -38,8 +42,12 @@ def test_watchdog_skips_already_terminal_task(tmp_path):
with _patch_trace_dir(tmp_path), patch.object(spawn_tool, "_kill_process_tree") as kill:
assert spawn_tool._watch_spawn(
trace_id, pid=999999, queue_ticket="t", conversation_id=None,
deadline_seconds=0.2, poll_interval=0.05,
trace_id,
pid=999999,
queue_ticket="t",
conversation_id=None,
deadline_seconds=0.2,
poll_interval=0.05,
)
kill.assert_not_called()
@@ -56,8 +64,12 @@ def test_watchdog_kills_and_fails_hung_runner(tmp_path):
patch.object(spawn_tool, "notify") as notifier,
):
assert not spawn_tool._watch_spawn(
trace_id, pid=424242, queue_ticket="ticket-1", conversation_id="conv-1",
deadline_seconds=0.2, poll_interval=0.05,
trace_id,
pid=424242,
queue_ticket="ticket-1",
conversation_id="conv-1",
deadline_seconds=0.2,
poll_interval=0.05,
)
kill.assert_called_once_with(424242)
@@ -91,7 +103,11 @@ def test_watchdog_tolerates_logs_appearing_at_deadline(tmp_path):
patch.object(spawn_tool, "_kill_process_tree") as kill,
):
assert spawn_tool._watch_spawn(
trace_id, pid=999999, queue_ticket="t", conversation_id=None,
deadline_seconds=0.15, poll_interval=0.2,
trace_id,
pid=999999,
queue_ticket="t",
conversation_id=None,
deadline_seconds=0.15,
poll_interval=0.2,
)
kill.assert_not_called()
tests/unit/mcp/test_task_reconcile.py
+2 -6
View File
@@ -36,9 +36,7 @@ def no_persistence(monkeypatch):
monkeypatch.setattr(task_manager, "_find_data_engine_db", lambda: None)
monkeypatch.setattr(task_manager, "_session_tracked_by_lock", lambda _tid: False)
monkeypatch.setattr(task_manager.trace_store, "write_status", lambda *_a, **_k: None)
monkeypatch.setattr(
task_manager.trace_store, "update_trace_status", lambda *_a, **_k: None
)
monkeypatch.setattr(task_manager.trace_store, "update_trace_status", lambda *_a, **_k: None)
monkeypatch.setattr(task_manager, "notify", lambda *_a, **_k: None)
@@ -48,9 +46,7 @@ def _db_with_session(tmp_path, trace_id: str, status: str, pid: int | None = 424
conn.execute(
"CREATE TABLE sessions (session_id TEXT, status TEXT, pid INTEGER, start_time REAL)"
)
conn.execute(
"INSERT INTO sessions VALUES (?, ?, ?, ?)", (trace_id, status, pid, time.time())
)
conn.execute("INSERT INTO sessions VALUES (?, ?, ?, ?)", (trace_id, status, pid, time.time()))
conn.commit()
conn.close()
return str(db_path)
tests/unit/memory/test_context_policy.py
+5 -27
View File
@@ -81,15 +81,6 @@ GOLDEN_CALLS = {
min_summaries=len(STEPS),
last_n_detailed=1,
),
"failure_analyzer": lambda: build_plan_and_history(
TASK_PLAN,
STEPS,
HASH_B,
last_n_detailed=1,
strict_milestone_pruning=True,
recent_window_size=3,
for_failure_analyzer=True,
),
"planner": lambda: build_plan_and_history(
"",
STEPS,
@@ -129,20 +120,13 @@ GOLDEN_CALLS = {
}
POLICY_CALLS = {
"operator": lambda: build_history_for(
"operator", TASK_PLAN, STEPS, HASH_B, last_n_detailed=1
),
"operator": lambda: build_history_for("operator", TASK_PLAN, STEPS, HASH_B, last_n_detailed=1),
"operator_cold_start": lambda: build_history_for(
"operator_cold_start", TASK_PLAN, STEPS, HASH_B
),
"failure_analyzer": lambda: build_history_for(
"failure_analyzer", TASK_PLAN, STEPS, HASH_B
),
"planner": lambda: build_history_for("planner", "", STEPS, "default"),
"outputter": lambda: build_history_for("outputter", TASK_PLAN, STEPS, HASH_B),
"history_analyzer": lambda: build_history_for(
"history_analyzer", TASK_PLAN, STEPS, HASH_B
),
"history_analyzer": lambda: build_history_for("history_analyzer", TASK_PLAN, STEPS, HASH_B),
"diagnoser": lambda: build_history_for(
"diagnoser", TASK_PLAN, STEPS, HASH_B, keep_subgoal_hashes={HASH_B}
),
@@ -223,18 +207,14 @@ def _chunk_row():
def _transcript_cfg(enabled: bool):
return SimpleNamespace(
memory=SimpleNamespace(
transcript=SimpleNamespace(enabled=enabled), policies={}
)
memory=SimpleNamespace(transcript=SimpleNamespace(enabled=enabled), policies={})
)
def test_flag_off_output_is_byte_identical_even_with_engine():
engine = _FakeEngine([_chunk_row()])
with patch("artemis.config.load_agent_config", return_value=_transcript_cfg(False)):
with_engine = build_history_for(
"outputter", TASK_PLAN, STEPS, HASH_B, engine=engine
)
with_engine = build_history_for("outputter", TASK_PLAN, STEPS, HASH_B, engine=engine)
without_engine = build_history_for("outputter", TASK_PLAN, STEPS, HASH_B)
assert with_engine == without_engine
assert "[Chunk 1" not in with_engine
@@ -280,9 +260,7 @@ def test_chunk_view_none_never_loads_chunks():
# The cold-start policy declares chunk_view=None: even with an engine
# and the flag on, its output carries no chunk blocks (the restored
# F region must stay a pure step rendering).
out = build_history_for(
"operator_cold_start", TASK_PLAN, STEPS, HASH_B, engine=engine
)
out = build_history_for("operator_cold_start", TASK_PLAN, STEPS, HASH_B, engine=engine)
assert CONTEXT_POLICIES["operator_cold_start"].chunk_view is None
assert "[Chunk 1" not in out
tests/unit/memory/test_history_chunking.py
+92 -59
View File
@@ -225,26 +225,47 @@ def test_action_ledger_lines_carry_step_number_offset_action_and_result():
assert lines[1].startswith("- Step 2 (T+00:20): Tapped 'btn2' at [2, 2] -> Error: boom")
def test_action_ledger_renders_fa_recovery_sublines_indented():
def test_action_ledger_renders_incident_and_burst_on_the_step_line():
"""No repair agent: a blocked step's ledger line carries the execution
incident phrase, and a fast-action burst lists every member."""
burst = [
{"action": "tap", "target_text": "player", "coordinates": [5, 6]},
{"action": "tap", "target_text": "Skip", "coordinates": [7, 8]},
]
steps = [
_step(
3,
interleaved_events=[
{
"type": "tool_call",
"name": "_exec_click",
"args": {"target": [5, 6], "target_text": "Retry"},
"result": "ok",
action_taken=burst,
last_execution_result={
"status": "failed",
"burst": True,
"execution": [
{"action": "tap", "target_text": "player"},
{"action": "tap", "target_text": "Skip", "attempts": ["Error: rejected"]},
],
"incident": {
"kind": "exec_error",
"category": "general",
"reason": "Error: rejected",
"action": burst[1],
"action_description": "Tapped 'Skip' at [7, 8]",
"action_index": 1,
"burst_size": 2,
"consecutive_failures": 1,
},
{"type": "tool_call", "name": "report_failure_analysis", "args": {}, "result": ""},
],
},
)
]
text = build_action_ledger(steps, SESSION_START)
lines = text.splitlines()
assert lines[0].startswith("- Step 3 ")
assert lines[1].startswith(" FA: Tapped 'Retry'")
assert "report_failure_analysis" not in text
assert len(lines) == 1
assert lines[0].startswith(
"- Step 3 (T+00:30): Fast-action burst (2 actions, unvetted):"
" Tapped 'player' at [5, 6] -> Tapped 'Skip' at [7, 8] -> Error: Execution failed"
" (general, consecutive failure #1) on burst action 2/2 `Tapped 'Skip' at [7, 8]`:"
" Error: rejected"
)
assert "FA:" not in text
def test_action_ledger_preserves_injected_instruction_verbatim():
@@ -272,11 +293,20 @@ def test_action_ledger_minimal_width_keeps_step_and_time_addressability():
def test_interval_coverage_requires_seamless_union():
ok = [{"start_step": 3, "end_step": 5, "text": "a"}, {"start_step": 6, "end_step": 6, "text": "b"}]
ok = [
{"start_step": 3, "end_step": 5, "text": "a"},
{"start_step": 6, "end_step": 6, "text": "b"},
]
assert validate_interval_coverage(ok, 3, 6)
gap = [{"start_step": 3, "end_step": 4, "text": "a"}, {"start_step": 6, "end_step": 6, "text": "b"}]
gap = [
{"start_step": 3, "end_step": 4, "text": "a"},
{"start_step": 6, "end_step": 6, "text": "b"},
]
assert not validate_interval_coverage(gap, 3, 6)
overlap = [{"start_step": 3, "end_step": 5, "text": "a"}, {"start_step": 5, "end_step": 6, "text": "b"}]
overlap = [
{"start_step": 3, "end_step": 5, "text": "a"},
{"start_step": 5, "end_step": 6, "text": "b"},
]
assert not validate_interval_coverage(overlap, 3, 6)
short = [{"start_step": 3, "end_step": 5, "text": "a"}]
assert not validate_interval_coverage(short, 3, 6)
@@ -401,9 +431,7 @@ def test_segment_size_threshold_triggers_without_milestone():
# Single milestone, but the open segment exceeded max_steps.
assert len(chunker.chunks) >= 1
assert chunker.chunks[0].start_step_number == 1
total = sum(
c.end_step_number - c.start_step_number + 1 for c in chunker.chunks
)
total = sum(c.end_step_number - c.start_step_number + 1 for c in chunker.chunks)
assert total >= 4
assert len(ledger.unchunked_turns()) >= 2 # floor respected
@@ -434,9 +462,7 @@ def test_soft_threshold_compresses_oldest_open_segment():
def test_min_active_floor_blocks_all_triggers():
steps = [_step(i, "hash-a" if i <= 2 else "hash-b") for i in range(1, 6)]
ledger, chunker, engine, _ = _make(
steps, min_active=5, max_steps=2, meter=lambda: 95_000
)
ledger, chunker, engine, _ = _make(steps, min_active=5, max_steps=2, meter=lambda: 95_000)
_run_turns(ledger, chunker, 1, 6, _hashes(2))
# Only 5 committed turns; the floor keeps everything raw.
assert len(chunker.chunks) == 0
@@ -461,8 +487,12 @@ def test_compression_boundary_never_splits_tool_call_pairs():
]
# And no orphan ToolMessage right after the frozen blocks.
first_active = next(
(m for m in rendered if isinstance(m, (AIMessage, ToolMessage, HumanMessage))
and "# CURRENT OBSERVATION" in str(getattr(m, "content", ""))),
(
m
for m in rendered
if isinstance(m, (AIMessage, ToolMessage, HumanMessage))
and "# CURRENT OBSERVATION" in str(getattr(m, "content", ""))
),
None,
)
assert first_active is not None
@@ -521,9 +551,7 @@ def test_chunk_block_renders_three_bands_in_order():
capsule.resolve("chunk:1-4", _capsule(1, 4))
_run_turns(ledger, chunker, 9, 12, lambda i: "hash-c") # second event
block = next(
str(m.content) for m in ledger.frozen_blocks if "[Chunk 1" in str(m.content)
)
block = next(str(m.content) for m in ledger.frozen_blocks if "[Chunk 1" in str(m.content))
# §3.3 structure: header, then ① → ② → ③ strictly in order.
i1 = block.index("① Synopsis & effects")
i2 = block.index("② Compressed step summary")
@@ -560,7 +588,14 @@ def test_checkpoint_annotation_bumps_version_and_rerenders_at_next_event():
annotated = chunker.annotate_from_checkpoint(
"hash-a",
[{"kind": "verify", "item_text": "logged in", "status": "failed", "evidence": "no session"}],
[
{
"kind": "verify",
"item_text": "logged in",
"status": "failed",
"evidence": "no session",
}
],
)
assert annotated
assert chunk.version == version_before + 1
@@ -650,9 +685,7 @@ def test_era_merge_keeps_ledgers_and_merges_headers():
merged_eras = [e for e in chunker.eras if not e.recall_only]
assert merged_eras
era_text = "\n".join(
str(m.content)
for m in ledger.frozen_blocks
if "① Merged synopsis" in str(m.content)
str(m.content) for m in ledger.frozen_blocks if "① Merged synopsis" in str(m.content)
)
assert "① Merged synopsis (structured fields, set-merged)" in era_text
assert "② Segment titles" in era_text
@@ -787,9 +820,7 @@ def test_independent_max_eras_cap_decouples_from_max_chunks():
def test_hard_threshold_renders_l3_snapshot_with_minimal_step_index():
steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 13)]
meter = {"value": None}
ledger, chunker, _, capsule = _make(
steps, min_active=2, meter=lambda: meter["value"]
)
ledger, chunker, _, capsule = _make(steps, min_active=2, meter=lambda: meter["value"])
_run_turns(ledger, chunker, 1, 8, _hashes(4))
capsule.resolve("chunk:1-4", _capsule(1, 4))
@@ -824,11 +855,7 @@ def test_step_addressability_survives_every_compression_level():
if not era.recall_only
for c in era.chunks
for n in range(c.start_step_number, c.end_step_number + 1)
] + [
n
for c in chunker.chunks
for n in range(c.start_step_number, c.end_step_number + 1)
]
] + [n for c in chunker.chunks for n in range(c.start_step_number, c.end_step_number + 1)]
assert covered_steps
for n in covered_steps:
assert f"- Step {n} (T+" in frozen, f"step {n} lost its addressable line"
@@ -905,8 +932,7 @@ async def test_capsule_lens_falls_back_to_secondary_model_on_provider_outage():
async def ainvoke(self, messages):
self.calls += 1
raise Exception(
"503 Service Unavailable. This model is currently experiencing"
" high demand."
"503 Service Unavailable. This model is currently experiencing high demand."
)
class HealthyFallback:
@@ -965,9 +991,7 @@ def test_capsule_fallback_model_resolution_google_only_and_not_primary():
def ctx_with(provider, model):
return SimpleNamespace(
llm_config=SimpleNamespace(
summarizer=SimpleNamespace(
fallback=SimpleNamespace(provider=provider, model=model)
)
summarizer=SimpleNamespace(fallback=SimpleNamespace(provider=provider, model=model))
)
)
@@ -978,9 +1002,7 @@ def test_capsule_fallback_model_resolution_google_only_and_not_primary():
# Non-google fallbacks cannot ride the raw google model path.
assert mgr._resolve_capsule_fallback_model(ctx_with("openai", "gpt-4o-mini")) is None
# A fallback identical to the primary adds nothing.
assert (
mgr._resolve_capsule_fallback_model(ctx_with("google", "gemini-3.7-flash")) is None
)
assert mgr._resolve_capsule_fallback_model(ctx_with("google", "gemini-3.7-flash")) is None
@pytest.mark.asyncio
@@ -1021,14 +1043,12 @@ def test_soft_pressure_alone_never_swaps_unready_segment():
correctness over tokens — the original text stays until the header lands."""
steps = [_step(i) for i in range(1, 9)]
meter = {"value": 75_000} # ≥ 0.7 * 100k, < 0.9 * 100k
ledger, chunker, _, _ = _make(
steps, min_active=2, max_steps=100, meter=lambda: meter["value"]
)
ledger, chunker, _, _ = _make(steps, min_active=2, max_steps=100, meter=lambda: meter["value"])
_run_turns(ledger, chunker, 1, 8)
assert len(chunker.awaiting_chunks) >= 1 # soft pressure closed a portion
assert chunker.chunks == () # ...but nothing swapped
assert ledger.frozen_blocks == () # original text retained
assert chunker.chunks == () # ...but nothing swapped
assert ledger.frozen_blocks == () # original text retained
def test_failed_capsule_redispatches_and_retains_original():
@@ -1048,8 +1068,8 @@ def test_failed_capsule_redispatches_and_retains_original():
ledger.render([_observation(9)])
assert capsule.submitted.count(key) == 2 # re-dispatched
assert chunker.chunks == () # still not swapped
assert ledger.frozen_blocks == () # original text retained
assert chunker.chunks == () # still not swapped
assert ledger.frozen_blocks == () # original text retained
def test_hard_threshold_force_swaps_pending_chunks_into_l3():
@@ -1093,15 +1113,11 @@ def test_capsule_note_coverage_is_machine_checked():
"steps": [
{
"step_number": 3,
"note_writes": [
{"tool": "save_note", "key": "login_flow", "gist": "code entry"}
],
"note_writes": [{"tool": "save_note", "key": "login_flow", "gist": "code entry"}],
},
{
"step_number": 5,
"note_writes": [
{"tool": "append_note", "key": "prices", "gist": "totals"}
],
"note_writes": [{"tool": "append_note", "key": "prices", "gist": "totals"}],
},
],
}
@@ -1143,3 +1159,20 @@ async def test_lens_backed_service_uses_lens_render():
await service.flush()
assert service.get_summary("k1") == "summary-for-k1"
assert not service.has_failed("k1")
def test_chunk_lists_every_step_of_a_multi_action_turn():
"""A Flash turn that executed several actions records one step per action
and registers them all on the committed turn (``step_keys``); the chunk's
band-③ ledger must list each of them, never only the turn's first step."""
steps = [_step(i) for i in range(1, 4)]
_ledger, chunker, _engine, _capsule = _make(steps)
turns = [
{"step_key": "s1", "step_keys": ["s1", "s2"], "start": 0, "end": 4},
{"step_key": "s3", "start": 4, "end": 7}, # legacy shape: step_key only
]
chunk = chunker._create_chunk(turns, None, {s["step_id"]: s for s in steps})
assert chunk is not None
assert chunk.source_step_ids == ["s1", "s2", "s3"]
assert chunk.start_step_number == 1 and chunk.end_step_number == 3
assert "- Step 2 (T+00:20)" in chunk.band3
tests/unit/memory/test_transcript_ledger.py
+44 -20
View File
@@ -85,9 +85,7 @@ def _fingerprint(msg) -> str:
def _has_image(msg) -> bool:
return any(
isinstance(b, dict) and b.get("type") in ("image_url", "image") for b in msg.content
)
return any(isinstance(b, dict) and b.get("type") in ("image_url", "image") for b in msg.content)
def test_format_session_offset():
@@ -139,8 +137,7 @@ def test_validator_result_message_carries_session_offset():
result_messages = [
m
for m in ledger.active_messages
if isinstance(m, HumanMessage)
and str(m.content).find(EXECUTION_RESULT_MARKER) >= 0
if isinstance(m, HumanMessage) and str(m.content).find(EXECUTION_RESULT_MARKER) >= 0
]
assert len(result_messages) == 1
text = result_messages[0].content[0]["text"]
@@ -153,9 +150,7 @@ def test_validator_result_skipped_for_actionless_turn():
ledger = TranscriptLedger(step_memory=_service())
_play_turn(ledger, 1)
_play_turn(ledger, 2, prev_key="step-1", prev_result=None)
assert not any(
EXECUTION_RESULT_MARKER in str(m.content) for m in ledger.active_messages
)
assert not any(EXECUTION_RESULT_MARKER in str(m.content) for m in ledger.active_messages)
def test_old_turn_recitation_and_ui_list_stripped_at_depth_1():
@@ -197,18 +192,14 @@ def test_pending_grace_then_placeholder_never_backfilled():
service = _service()
service._step_inputs["step-1"] = {"step_number": 1} # pending job, no summary
ledger = TranscriptLedger(
step_memory=service, image_scrub_depth=2, pending_grace_steps=1
)
ledger = TranscriptLedger(step_memory=service, image_scrub_depth=2, pending_grace_steps=1)
# Enough turns to push turn 1 past K + grace.
for i in range(1, 7):
_play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
first_obs = ledger.active_messages[0]
assert not _has_image(first_obs)
assert "[visual summary pending; evidence at DataEngine step 1]" in str(
first_obs.content
)
assert "[visual summary pending; evidence at DataEngine step 1]" in str(first_obs.content)
frozen = _fingerprint(first_obs)
# A late summary must never mutate the frozen message.
@@ -222,17 +213,13 @@ def test_failed_summary_becomes_unavailable_placeholder():
service._step_inputs["step-1"] = {"step_number": 1}
service._failed.add("step-1")
ledger = TranscriptLedger(
step_memory=service, image_scrub_depth=2, pending_grace_steps=5
)
ledger = TranscriptLedger(step_memory=service, image_scrub_depth=2, pending_grace_steps=5)
for i in range(1, 4):
_play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
first_obs = ledger.active_messages[0]
assert not _has_image(first_obs)
assert "[visual summary unavailable; evidence at DataEngine step 1]" in str(
first_obs.content
)
assert "[visual summary unavailable; evidence at DataEngine step 1]" in str(first_obs.content)
def test_tool_call_response_pairs_are_never_split():
@@ -296,3 +283,40 @@ def test_no_ago_wording_in_ledger_output():
)
blob = " ".join(str(m.content) for m in ledger.active_messages)
assert " ago" not in blob
# ---------------------------------------------------------------------------
# Session-start anchoring and multi-step turns (Flash profile reuse)
# ---------------------------------------------------------------------------
def test_session_start_anchors_offsets_to_the_data_engine_clock():
"""With a ``session_start`` epoch the ledger reads the wall clock, so the
``T+mm:ss`` labels are relative to the recorded session start (the same
origin as the chunk ledger lines and the video analyzer's timeline)."""
now = {"t": 1000.0}
ledger = TranscriptLedger(step_memory=_service(), clock=lambda: now["t"], session_start=940.0)
assert ledger.elapsed_label() == "T+01:00"
now["t"] = 1075.5
assert ledger.elapsed_label() == "T+02:15"
def test_without_session_start_the_ledger_keeps_its_own_origin():
now = {"t": 500.0}
ledger = TranscriptLedger(step_memory=_service(), clock=lambda: now["t"])
assert ledger.elapsed_label() == "T+00:00"
now["t"] = 512.0
assert ledger.elapsed_label() == "T+00:12"
def test_commit_records_every_step_key_of_a_multi_action_turn():
ledger = TranscriptLedger(step_memory=_service())
ledger.stage_turn(_turn(1, tool_call_count=2))
ledger.commit_staged(step_key="s1", extra_step_keys=["s2", "s1", "s3"])
turn = ledger.unchunked_turns()[0]
assert turn["step_key"] == "s1"
assert turn["step_keys"] == ["s1", "s2", "s3"]
ledger.stage_turn(_turn(2))
ledger.commit_staged(step_key=None)
assert ledger.unchunked_turns()[1]["step_keys"] == []
tests/unit/runtime/test_cancel_requests.py
+93
View File
@@ -0,0 +1,93 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import json
import time
import pytest
from artemis.runtime import cancel_requests as cr
@pytest.fixture(autouse=True)
def isolated_dir(tmp_path, monkeypatch):
directory = tmp_path / "cancel-requests"
monkeypatch.setattr(cr, "get_temp_dir", lambda _subfolder=None: directory)
return directory
def test_session_marker_roundtrip(isolated_dir):
assert not cr.is_cancel_requested(session_id="s-1")
written = cr.request_cancel(session_id="s-1", reason="test")
assert len(written) == 1
assert written[0].parent == isolated_dir
assert cr.is_cancel_requested(session_id="s-1")
cr.clear_cancel_request(session_id="s-1")
assert not cr.is_cancel_requested(session_id="s-1")
def test_pid_marker_ignores_recycled_pid():
cr.request_cancel(pid=4242, process_created_at=1000.0)
assert cr.is_cancel_requested(pid=4242, process_created_at=1000.4)
assert not cr.is_cancel_requested(pid=4242, process_created_at=2000.0)
# A marker without a creation time applies to whoever holds the pid.
cr.request_cancel(pid=4343)
assert cr.is_cancel_requested(pid=4343, process_created_at=5.0)
def test_expired_markers_are_ignored_and_purged():
path = cr.request_cancel(session_id="old")[0]
payload = json.loads(path.read_text(encoding="utf-8"))
payload["requested_at"] = time.time() - cr.MARKER_TTL_SECONDS - 5
path.write_text(json.dumps(payload), encoding="utf-8")
assert not cr.is_cancel_requested(session_id="old")
assert cr.purge_expired() == 1
assert not path.exists()
def test_unreadable_pid_marker_is_not_trusted(isolated_dir):
isolated_dir.mkdir(parents=True, exist_ok=True)
(isolated_dir / "pid-77.cancel").write_text("{not json", encoding="utf-8")
assert not cr.is_cancel_requested(pid=77, process_created_at=1.0)
@pytest.mark.asyncio
async def test_watch_delivers_once_and_clears_marker():
delivered: list[int] = []
watcher = asyncio.create_task(
cr.watch_for_cancel_request(
lambda: delivered.append(1),
session_id="w-1",
pid=0,
poll_seconds=0.05,
)
)
await asyncio.sleep(0.12)
assert not delivered
assert not watcher.done()
cr.request_cancel(session_id="w-1")
assert await asyncio.wait_for(watcher, timeout=2) is True
assert delivered == [1]
assert not cr.is_cancel_requested(session_id="w-1")
tests/unit/runtime/test_daemon_client.py
+52 -1
View File
@@ -131,6 +131,58 @@ def test_submit_task_to_daemon():
assert data["conversation_id"] == "conv-123"
def _daemon_ok_response(body: bytes) -> MagicMock:
mock_resp = MagicMock()
mock_resp.status = 200
mock_resp.read.return_value = body
mock_resp.__enter__.return_value = mock_resp
return mock_resp
def test_submit_task_to_daemon_forwards_pro_tuning_knobs():
mock_resp = _daemon_ok_response(b'{"status": "queued", "tasks": [{"session_id": "s1"}]}')
with patch("urllib.request.urlopen", return_value=mock_resp) as mock_urlopen:
submit_task_to_daemon(
"audit goal",
profile="pro",
verification_level="strict",
explorer_mode="ultra",
)
data = json.loads(mock_urlopen.call_args[0][0].data.decode("utf-8"))
# JSON field names match RunRequest on /api/run.
assert data["verification_level"] == "strict"
assert data["explorer_mode"] == "ultra"
def test_submit_task_to_daemon_pro_tuning_defaults_to_null():
mock_resp = _daemon_ok_response(b'{"status": "queued", "tasks": [{"session_id": "s1"}]}')
with patch("urllib.request.urlopen", return_value=mock_resp) as mock_urlopen:
submit_task_to_daemon("plain goal", profile="flash")
data = json.loads(mock_urlopen.call_args[0][0].data.decode("utf-8"))
assert data["verification_level"] is None
assert data["explorer_mode"] is None
def test_submit_batch_to_daemon_forwards_pro_tuning_knobs():
from artemis.runtime.daemon_client import submit_batch_to_daemon
mock_resp = _daemon_ok_response(
b'{"status": "queued", "tasks": [{"session_id": "a"}, {"session_id": "b"}]}'
)
with patch("urllib.request.urlopen", return_value=mock_resp) as mock_urlopen:
res = submit_batch_to_daemon(
["goal a", "goal b"],
profile="pro",
verification_level="checkpoints",
explorer_mode="pro",
)
data = json.loads(mock_urlopen.call_args[0][0].data.decode("utf-8"))
assert res is not None and len(res["tasks"]) == 2
assert data["goals"] == ["goal a", "goal b"]
assert data["verification_level"] == "checkpoints"
assert data["explorer_mode"] == "pro"
def test_stop_task_on_daemon():
from artemis.runtime.daemon_client import stop_task_on_daemon
@@ -148,4 +200,3 @@ def test_stop_task_on_daemon():
mock_resp.read.return_value = b'{"status": "no_running_task"}'
with patch("urllib.request.urlopen", return_value=mock_resp):
assert stop_task_on_daemon("non-existent-sess") is False
tests/unit/runtime/test_device_lock.py
+1 -3
View File
@@ -197,9 +197,7 @@ def test_pending_reservation_at_queue_head_does_not_block_other_devices():
pending_ticket = DeviceExecutionLock.reserve("unclaimed submission")
# device-alpha is a known idle device (a queued submission targets it), so
# the pending ticket parks there; device-beta must stay schedulable.
alpha_ticket = DeviceExecutionLock.reserve(
"task for device alpha", device_id="device-alpha"
)
alpha_ticket = DeviceExecutionLock.reserve("task for device alpha", device_id="device-alpha")
try:
beta = DeviceExecutionLock("device-beta", "task for device beta")
# Must acquire immediately even though the older pending ticket is at
tests/unit/runtime/test_server_process_lifecycle.py
+6 -10
View File
@@ -202,9 +202,7 @@ def test_stop_server_terminates_tree(monkeypatch):
lambda: 1,
)
reconcile = MagicMock(return_value=1)
monkeypatch.setattr(
"artemis.runtime.server_lifecycle._reconcile_orphaned_sessions", reconcile
)
monkeypatch.setattr("artemis.runtime.server_lifecycle._reconcile_orphaned_sessions", reconcile)
success, msg, stopped = stop_server(8000)
assert success is True
@@ -218,9 +216,7 @@ def test_stop_server_terminates_tree(monkeypatch):
@pytest.mark.asyncio
async def test_api_server_status():
"""Verify GET /api/system/server-status returns server runtime metadata."""
async with AsyncClient(
transport=ASGITransport(app=app), base_url="http://localhost"
) as client:
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://localhost") as client:
response = await client.get("/api/system/server-status")
assert response.status_code == 200
data = response.json()
@@ -243,7 +239,9 @@ async def test_api_restart_endpoint():
assert response.status_code == 200
data = response.json()
assert data["status"] == "restarting"
assert "reconnection" in data["message"].lower() or "restarting" in data["message"].lower()
assert (
"reconnection" in data["message"].lower() or "restarting" in data["message"].lower()
)
mock_thread.assert_called_once()
mock_instance.start.assert_called_once()
@@ -254,9 +252,7 @@ async def test_api_shutdown_endpoint():
fake_server = MagicMock(should_exit=False)
app.state.uvicorn_server = fake_server
async with AsyncClient(
transport=ASGITransport(app=app), base_url="http://localhost"
) as client:
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://localhost") as client:
denied = await client.post("/api/system/shutdown")
assert denied.status_code == 403
tests/unit/runtime/test_trace_store.py
+21 -12
View File
@@ -1,4 +1,4 @@
# Copyright 2026 Google LLC
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -172,7 +172,7 @@ def test_update_on_corrupt_status_logs_and_drops(temp_trace_env, caplog):
assert not any("Dropping status update" in rec.message for rec in caplog.records)
def test_concurrent_read_modify_write_does_not_lose_updates(temp_trace_env):
def test_concurrent_read_modify_write_does_not_lose_updates(temp_trace_env, caplog):
"""Interleaved RMW updates from two writers must not lose each other's fields."""
trace_id = str(uuid.uuid4())
trace_store.init_trace(trace_id, "Concurrency", "Flash")
@@ -182,28 +182,37 @@ def test_concurrent_read_modify_write_does_not_lose_updates(temp_trace_env):
def set_errors():
try:
for i in range(iterations):
assert trace_store.update_trace_status(
trace_id, "running", error=f"err-{i}"
) is not None
assert (
trace_store.update_trace_status(trace_id, "running", error=f"err-{i}")
is not None
)
except BaseException as exc: # noqa: BLE001 - surfaced in the main thread
failures.append(exc)
def set_serials():
try:
for i in range(iterations):
assert trace_store.update_trace_device_serial(
trace_id, f"serial-{i}"
) is not None
assert trace_store.update_trace_device_serial(trace_id, f"serial-{i}") is not None
except BaseException as exc: # noqa: BLE001 - surfaced in the main thread
failures.append(exc)
threads = [threading.Thread(target=set_errors), threading.Thread(target=set_serials)]
for t in threads:
t.start()
for t in threads:
t.join(timeout=30.0)
with caplog.at_level(logging.WARNING, logger="artemis.runtime.trace_store"):
for t in threads:
t.start()
for t in threads:
t.join(timeout=30.0)
assert failures == []
# The lock is bounded by design: after _LOCK_TIMEOUT_SECONDS a writer
# proceeds last-writer-wins (with a WARNING) rather than deadlocking. On a
# heavily loaded machine that documented path can be taken, and then a
# lost field is expected rather than a serialization bug.
lock_degraded = [rec.message for rec in caplog.records if "trace status lock" in rec.message]
if lock_degraded:
pytest.skip(
f"status lock degraded under load, serialization not exercised: {lock_degraded[0]}"
)
final = trace_store.read_status(trace_id)
assert final is not None
# Every serialized RMW preserves the other writer's latest field, so both
tests/unit/sdk/test_agent_config_builder_tuning.py
+123
View File
@@ -0,0 +1,123 @@
"""SDK entry point for the Pro-profile tuning knobs.
``AgentConfigBuilder.with_verification_level`` mirrors ``--verification-level``
and ``with_pro_config(verification_level=..., explorer_mode=...)`` carries both
knobs in one call; both resolve through the single preset table in
``artemis.config``.
"""
import pytest
from artemis.config import VERIFICATION_LEVEL_PRESETS, checker_overrides_for_level
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
def test_with_verification_level_off_disables_the_checker():
cfg = AgentConfigBuilder().with_verification_level("off").build()
assert cfg.disable_checker is True
def test_with_verification_level_final_keeps_factory_layering():
cfg = AgentConfigBuilder().with_verification_level("final").build()
assert cfg.disable_checker is False
assert cfg.disable_midway_checks is True
assert cfg.disable_final_check is False
def test_with_verification_level_checkpoints_enables_midway_checks():
cfg = AgentConfigBuilder().with_verification_level("checkpoints").build()
assert cfg.disable_checker is False
assert cfg.disable_midway_checks is False
assert cfg.disable_final_check is False
# Unspecified fields keep the artemis.jsonc values.
assert cfg.assert_failure_policy == "continue"
def test_with_verification_level_strict_applies_the_full_preset():
cfg = AgentConfigBuilder().with_verification_level("strict").build()
preset = VERIFICATION_LEVEL_PRESETS["strict"]
assert cfg.disable_checker is False
assert cfg.disable_midway_checks is False
assert cfg.disable_final_check is False
assert cfg.assert_failure_policy == preset["assert_failure_policy"]
assert cfg.checkpoint_max_repairs == preset["checkpoint_max_repairs"]
assert cfg.final_check_max_attempts == preset["final_check_max_attempts"]
assert cfg.checker_max_iterations == preset["max_iterations"]
def test_with_verification_level_is_case_and_whitespace_insensitive():
cfg = AgentConfigBuilder().with_verification_level(" STRICT ").build()
assert cfg.assert_failure_policy == "halt"
def test_with_verification_level_rejects_unknown_preset():
builder = AgentConfigBuilder()
with pytest.raises(ValueError, match="Unknown verification level"):
builder.with_verification_level("paranoid")
# The builder is untouched by a rejected call.
assert builder.build().disable_checker is False
def test_with_verification_level_matches_the_shared_preset_table():
for level in VERIFICATION_LEVEL_PRESETS:
cfg = AgentConfigBuilder().with_verification_level(level).build()
overrides = checker_overrides_for_level(level)
assert cfg.disable_checker is (not overrides["enabled"])
if "midway_checks" in overrides:
assert cfg.disable_midway_checks is (not overrides["midway_checks"])
def test_with_verification_level_is_chainable_and_explicit_checker_wins():
cfg = AgentConfigBuilder().with_verification_level("strict").with_checker(enabled=False).build()
assert cfg.disable_checker is True
# The preset's other fields survive the later master switch.
assert cfg.assert_failure_policy == "halt"
def test_with_pro_config_carries_both_knobs():
cfg = (
AgentConfigBuilder()
.with_pro_config(verification_level="checkpoints", explorer_mode="ultra")
.build()
)
assert cfg.disable_checker is False
assert cfg.disable_midway_checks is False
assert cfg.explorer.pro_mode == "ultra"
# Flash-profile perception is untouched by the Pro knob.
assert cfg.explorer.flash_mode == "flash"
def test_with_pro_config_checker_switch_wins_over_verification_level():
cfg = AgentConfigBuilder().with_pro_config(verification_level="strict", checker=False).build()
assert cfg.disable_checker is True
def test_with_explorer_pro_mode_is_the_explorer_mode_knob(monkeypatch):
monkeypatch.delenv("ARTEMIS_EXPLORER_VERSION", raising=False)
cfg = AgentConfigBuilder().with_explorer(pro_mode="pro").build()
assert cfg.explorer.pro_mode == "pro"
assert cfg.get_explorer_version(agent_name=None, profile="pro") == "pro"
# The shipped per-agent override is empty, so the Pro agents follow the knob.
assert cfg.get_explorer_version(agent_name="operator") == "pro"
assert cfg.get_explorer_version(agent_name="validator") == "pro"
def test_with_flash_config_explorer_mode_is_the_flash_profile_knob(monkeypatch):
monkeypatch.delenv("ARTEMIS_EXPLORER_VERSION", raising=False)
cfg = AgentConfigBuilder().with_flash_config(explorer_mode="pro").build()
assert cfg.flash.explorer_mode == "pro"
assert cfg.explorer.flash_mode == "pro"
assert cfg.get_explorer_version(agent_name="flash") == "pro"
# Pro-profile perception is untouched by the Flash knob.
assert cfg.get_explorer_version(agent_name="operator") == "flash"
def test_with_explorer_versions_is_an_advanced_per_agent_override(monkeypatch):
monkeypatch.delenv("ARTEMIS_EXPLORER_VERSION", raising=False)
cfg = (
AgentConfigBuilder()
.with_explorer(pro_mode="flash", versions={"validator": "ultra"})
.build()
)
assert cfg.get_explorer_version(agent_name="validator") == "ultra"
assert cfg.get_explorer_version(agent_name="operator") == "flash"
tests/unit/sdk/test_agent_external_cancel.py
+46
View File
@@ -0,0 +1,46 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""A cancel marker dropped by another process must cancel the Agent's task."""
import asyncio
import pytest
from artemis.runtime import cancel_requests
from artemis.sdk.agent import Agent
@pytest.mark.asyncio
async def test_external_cancel_marker_cancels_running_task(tmp_path, monkeypatch):
monkeypatch.setattr(cancel_requests, "get_temp_dir", lambda _subfolder=None: tmp_path / "c")
monkeypatch.setenv("ARTEMIS_CANCEL_POLL_SECONDS", "0.05")
agent = Agent.__new__(Agent)
agent._session_id = "sess-cancel"
async def _work() -> None:
await asyncio.sleep(30)
agent._current_task = asyncio.create_task(_work())
watcher = asyncio.create_task(agent._watch_external_cancel("demo"))
await asyncio.sleep(0.15)
assert not agent._current_task.done()
cancel_requests.request_cancel(session_id="sess-cancel", reason="stop from UI")
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(agent._current_task, timeout=2)
await asyncio.wait_for(watcher, timeout=2)
assert not cancel_requests.is_cancel_requested(session_id="sess-cancel")
tests/unit/sdk/test_agent_trace_finalization.py
+2 -4
View File
@@ -94,8 +94,7 @@ async def test_secure_keyguard_is_rejected_without_guessing_credentials():
agent._device_context = MagicMock(device_id="device-123")
agent._adb_client = MagicMock()
agent._adb_client.device.return_value.shell.return_value = (
'User "Owner" (id=0): deviceLocked=1\n'
'User "Work" (id=10): deviceLocked=1\n'
'User "Owner" (id=0): deviceLocked=1\nUser "Work" (id=10): deviceLocked=1\n'
)
with pytest.raises(AgentError, match="Unlock the device manually"):
@@ -108,8 +107,7 @@ async def test_locked_work_profile_does_not_block_unlocked_device_owner():
agent._device_context = MagicMock(device_id="device-123")
agent._adb_client = MagicMock()
agent._adb_client.device.return_value.shell.return_value = (
'User "Owner" (id=0): deviceLocked=0\n'
'User "Work" (id=10): deviceLocked=1\n'
'User "Owner" (id=0): deviceLocked=0\nUser "Work" (id=10): deviceLocked=1\n'
)
await agent._ensure_device_unlocked()
tests/unit/test_await.py
+6 -10
View File
@@ -12,19 +12,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import pytest
async def f():
return [1, 2]
async def m():
try:
val = await f()[-1]
print(val)
except Exception as e:
print(repr(e))
asyncio.run(m())
@pytest.mark.asyncio
async def test_await_result_before_indexing():
"""Await the coroutine before indexing its returned list."""
val = (await f())[-1]
assert val == 2
tests/unit/test_cli.py
+103 -6
View File
@@ -46,6 +46,8 @@ def test_cli_run_help():
assert "--profile" in result.output
assert "--locked-app" in result.output
assert "--traces-path" in result.output
assert "--verification-level" in result.output
assert "--explorer-pro-mode" in result.output
def test_cli_batch_help():
@@ -54,6 +56,101 @@ def test_cli_batch_help():
assert result.exit_code == 0
assert "--file" in result.output
assert "--delay" in result.output
assert "--verification-level" in result.output
assert "--explorer-pro-mode" in result.output
def test_cli_batch_forwards_pro_tuning_in_standalone_mode(monkeypatch):
"""`artemis batch --standalone` threads both knobs into run_batch_tasks."""
import artemis.interfaces.cli.commands.batch as batch_module
captured: dict = {}
async def fake_run_batch_tasks(tasks, **kwargs):
captured["tasks"] = tasks
captured.update(kwargs)
monkeypatch.setattr(batch_module, "run_batch_tasks", fake_run_batch_tasks)
result = runner.invoke(
app,
[
"batch",
"--standalone",
"--profile",
"pro",
"--verification-level",
"strict",
"--explorer-pro-mode",
"ultra",
"Open Settings",
],
)
assert result.exit_code == 0, result.output
assert captured["tasks"] == ["Open Settings"]
assert captured["profile_name"] == "pro"
assert captured["verification_level"] == "strict"
assert captured["explorer_pro_mode"] == "ultra"
def test_cli_batch_forwards_pro_tuning_to_daemon(monkeypatch):
"""Daemon-routed batches carry the knobs as /api/run JSON fields."""
import artemis.runtime as runtime
monkeypatch.delenv("ARTEMIS_STANDALONE", raising=False)
captured: dict = {}
def fake_submit_batch(goals, **kwargs):
captured["goals"] = goals
captured.update(kwargs)
return {"tasks": [{"session_id": "sid-1", "goal": goals[0]}]}
monkeypatch.setattr(runtime, "ensure_daemon_running", lambda **_: (True, "http://x:1"))
monkeypatch.setattr(runtime, "submit_batch_to_daemon", fake_submit_batch)
monkeypatch.setattr(runtime, "wait_for_daemon_task", lambda *_, **__: {"status": "completed"})
result = runner.invoke(
app,
["batch", "--verification-level", "checkpoints", "--explorer-pro-mode", "pro", "Goal A"],
)
assert result.exit_code == 0, result.output
assert captured["goals"] == ["Goal A"]
assert captured["verification_level"] == "checkpoints"
assert captured["explorer_mode"] == "pro"
def test_run_batch_tasks_applies_pro_tuning_to_agent_config(monkeypatch):
"""The standalone batch runner applies the knobs on the AgentConfig builder."""
from unittest.mock import AsyncMock, MagicMock
import artemis.interfaces.cli.commands.batch as batch_module
fake_builder = MagicMock()
fake_builders = MagicMock()
fake_builders.AgentConfig.with_default_profile.return_value = fake_builder
fake_agent = MagicMock()
fake_agent.init = AsyncMock()
fake_agent.run_task = AsyncMock(return_value="ok")
fake_agent.clean = AsyncMock()
monkeypatch.setattr(batch_module, "initialize_llm_config", lambda: MagicMock())
monkeypatch.setattr(batch_module, "AgentProfile", MagicMock())
monkeypatch.setattr(batch_module, "Builders", fake_builders)
monkeypatch.setattr(batch_module, "Agent", MagicMock(return_value=fake_agent))
import asyncio
asyncio.run(
batch_module.run_batch_tasks(
["Goal A"],
profile_name="pro",
delay_seconds=0,
verification_level="strict",
explorer_pro_mode="ultra",
)
)
fake_builder.with_verification_level.assert_called_once_with("strict")
fake_builder.with_explorer.assert_called_once_with(pro_mode="ultra")
fake_agent.run_task.assert_awaited_once_with(goal="Goal A", profile="pro")
def test_cli_trace_help():
@@ -93,9 +190,7 @@ def test_cli_mcp_generate_config_antigravity():
assert server_config["disabledTools"] == []
assert "tools" not in server_config
legacy_config = _get_config_snippet("jetski", "python", "/project")["mcpServers"][
"artemis"
]
legacy_config = _get_config_snippet("jetski", "python", "/project")["mcpServers"]["artemis"]
assert set(legacy_config["tools"]) == {
"mobile_run_task",
"mobile_manage_task",
@@ -152,6 +247,7 @@ def test_cli_mcp_install_antigravity(tmp_path, monkeypatch):
jetski_file = tmp_path / ".gemini" / "jetski" / "mcp_config.json"
assert jetski_file.exists()
import json
data = json.loads(jetski_file.read_text())
assert "artemis" in data["mcpServers"]
assert "mobile_run_task" in data["mcpServers"]["artemis"]["tools"]
@@ -190,15 +286,14 @@ def test_cli_mcp_install_all(tmp_path, monkeypatch):
cursor_data = json.loads((tmp_path / ".cursor" / "mcp.json").read_text())
assert cursor_data["mcpServers"]["artemis"]["command"]
windsurf_data = json.loads(
(tmp_path / ".codeium" / "windsurf" / "mcp_config.json").read_text()
)
windsurf_data = json.loads((tmp_path / ".codeium" / "windsurf" / "mcp_config.json").read_text())
assert windsurf_data["mcpServers"]["artemis"]["command"]
claude_data = json.loads((tmp_path / ".claude.json").read_text())
assert claude_data["mcpServers"]["artemis"]["type"] == "stdio"
from artemis.interfaces.cli.commands.mcp import _get_vscode_user_dir
vscode_data = json.loads((_get_vscode_user_dir() / "mcp.json").read_text())
assert vscode_data["servers"]["artemis"]["type"] == "stdio"
assert "mcpServers" not in vscode_data
@@ -321,6 +416,7 @@ def test_cli_mcp_install_codex_preserves_config_and_is_idempotent(tmp_path, monk
assert result2.exit_code == 0
import tomllib
config_text = config_file.read_text(encoding="utf-8")
data = tomllib.loads(config_text)
assert data["model"] == "test-model"
@@ -361,6 +457,7 @@ def test_cli_mcp_install_jsonc_and_backup(tmp_path, monkeypatch):
result = runner.invoke(app, ["mcp", "--install", "cursor"])
assert result.exit_code == 0
import json
data = json.loads(mcp_json.read_text(encoding="utf-8"))
assert "test" in data["mcpServers"]
assert "artemis" in data["mcpServers"]
tests/unit/test_config_package.py
+115 -16
View File
@@ -172,15 +172,16 @@ def test_agent_config_loading():
"""Test AgentGlobalConfig parsing from agent_config.json / artemis.jsonc."""
agent_cfg = load_agent_config()
assert isinstance(agent_cfg, AgentGlobalConfig)
assert "operator" in agent_cfg.explorer_versions
# The per-agent override ships empty so the profile knobs decide; caching
# ships unset so each tier applies its own default (off pro, on ultra).
assert agent_cfg.explorer_versions == {}
assert agent_cfg.explorer.default_version == "flash"
assert agent_cfg.explorer.flash_mode == "flash"
assert agent_cfg.explorer.pro_mode == "flash"
assert agent_cfg.explorer.caching is True
assert agent_cfg.explorer.caching is None
assert "explorer" in agent_cfg.denylisted_tools
assert agent_cfg.video_analyzer.enable_ledger is True
assert agent_cfg.planner_validation.enabled is True
assert agent_cfg.planner_validation.similarity_threshold == 0.85
assert agent_cfg.committee.enabled is False
assert agent_cfg.committee.debate_rounds == 2
assert agent_cfg.checker.enabled is True
@@ -191,7 +192,7 @@ def test_agent_config_loading():
assert agent_cfg.checker.device_probes is True
assert agent_cfg.outputter.enabled is True
assert agent_cfg.outputter.force_synthesis is False
assert agent_cfg.flash.max_turns == 30
assert agent_cfg.flash.max_turns == 0
assert agent_cfg.flash.explorer_mode == "flash"
assert agent_cfg.pro.explorer.mode == "flash"
assert agent_cfg.pro.checker.enabled is True
@@ -255,20 +256,17 @@ def test_planner_validation_builder_and_milestones():
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
from artemis.utils.plan_grammar import milestones_changed, parse_plan
# Default builder inherits from artemis.jsonc (enabled=True, similarity_threshold=0.85)
# Default builder inherits from artemis.jsonc (enabled=True)
builder = AgentConfigBuilder()
cfg = builder.build()
assert cfg.disable_planner_validation is False
assert cfg.planner_validation_threshold == 0.85
# The dead similarity-threshold knob is gone: validation has no tunable
# trigger, every milestone text change is reviewed.
assert not hasattr(cfg, "planner_validation_threshold")
# Fluent enabling
cfg_enabled = (
AgentConfigBuilder()
.with_planner_validation(enabled=True, similarity_threshold=0.90)
.build()
)
cfg_enabled = AgentConfigBuilder().with_planner_validation(enabled=True).build()
assert cfg_enabled.disable_planner_validation is False
assert cfg_enabled.planner_validation_threshold == 0.90
# Fluent disabling
cfg_disabled = AgentConfigBuilder().with_disable_planner_validation(True).build()
@@ -326,9 +324,10 @@ async def test_committee_builder_and_graph_mounting():
graph_enabled = await get_graph(ctx_enabled)
op_node_enabled = graph_enabled.nodes.get("operator")
assert op_node_enabled is not None
# Verify ask_committee is mounted when enabled
# ask_committee is deliberately never mounted on the Operator (it sits outside
# the pre-decision / turn-ending tool contract), even when the flag is on.
op_tools_enabled = [t.name for t in op_node_enabled.bound.afunc.tools]
assert "ask_committee" in op_tools_enabled
assert "ask_committee" not in op_tools_enabled
def test_checker_builder_and_context_propagation():
@@ -450,13 +449,27 @@ def test_explorer_builder_and_resolution(monkeypatch):
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
from artemis.tools.explorer_tool import resolve_explorer_version
# Default builder inherits from artemis.jsonc (default="flash", flash_mode="flash", pro_mode="flash", caching=True)
monkeypatch.delenv("ARTEMIS_EXPLORER_VERSION", raising=False)
# Default builder inherits from artemis.jsonc (default="flash", flash_mode="flash",
# pro_mode="flash", caching unset, no per-agent override).
builder = AgentConfigBuilder()
cfg = builder.build()
assert cfg.explorer.default_version == "flash"
assert cfg.explorer.flash_mode == "flash"
assert cfg.explorer.pro_mode == "flash"
assert cfg.explorer.caching is True
assert cfg.explorer.caching is None
assert cfg.explorer_versions == {}
# With the shipped (empty) override the profile knobs actually win: the
# Pro agents follow pro_mode and the Flash runner follows flash_mode.
cfg_pro_ultra = AgentConfigBuilder().with_explorer(pro_mode="ultra").build()
assert cfg_pro_ultra.get_explorer_version(agent_name="operator") == "ultra"
assert cfg_pro_ultra.get_explorer_version(agent_name="validator") == "ultra"
assert cfg_pro_ultra.get_explorer_version(agent_name="flash") == "flash"
cfg_flash_pro = AgentConfigBuilder().with_explorer(flash_mode="pro").build()
assert cfg_flash_pro.get_explorer_version(agent_name="flash") == "pro"
assert cfg_flash_pro.get_explorer_version(agent_name="operator") == "flash"
# Fluent configuration with with_explorer
cfg_custom = (
@@ -709,3 +722,89 @@ def test_admin_console_config_facade_backward_compatibility():
assert DB_PATH == ac.DB_PATH
assert TRACES_PATH == ac.TRACES_PATH
assert callable(init_ls_address)
class TestVerificationLevelPresets:
"""The coarse Checker presets behind ``--verification-level``."""
def test_every_preset_is_a_valid_checker_override(self):
from artemis.config import (
DEFAULT_VERIFICATION_LEVEL,
VERIFICATION_LEVEL_PRESETS,
CheckerConfig,
checker_overrides_for_level,
)
assert DEFAULT_VERIFICATION_LEVEL in VERIFICATION_LEVEL_PRESETS
for level in VERIFICATION_LEVEL_PRESETS:
cfg = CheckerConfig(**checker_overrides_for_level(level))
assert isinstance(cfg, CheckerConfig)
def test_presets_form_a_monotonic_ladder(self):
from artemis.config import CheckerConfig, checker_overrides_for_level
off = CheckerConfig(**checker_overrides_for_level("off"))
final = CheckerConfig(**checker_overrides_for_level("final"))
checkpoints = CheckerConfig(**checker_overrides_for_level("checkpoints"))
strict = CheckerConfig(**checker_overrides_for_level("strict"))
assert off.enabled is False
assert final.enabled and final.final_check and not final.midway_checks
# ``final`` is the factory layering: identical to a default CheckerConfig.
assert final == CheckerConfig()
assert checkpoints.midway_checks and checkpoints.final_check
assert checkpoints.assert_failure_policy == "continue"
assert strict.midway_checks and strict.final_check
assert strict.assert_failure_policy == "halt"
assert strict.checkpoint_max_repairs > checkpoints.checkpoint_max_repairs
assert strict.final_check_max_attempts > checkpoints.final_check_max_attempts
assert strict.max_iterations > checkpoints.max_iterations
def test_level_lookup_is_case_and_whitespace_insensitive(self):
from artemis.config import checker_overrides_for_level
assert checker_overrides_for_level(" STRICT ") == checker_overrides_for_level("strict")
# A copy is returned so callers cannot mutate the shared preset table.
overrides = checker_overrides_for_level("off")
overrides["enabled"] = True
assert checker_overrides_for_level("off")["enabled"] is False
@pytest.mark.parametrize("bad", [None, "", "maximum", "ultra"])
def test_unknown_level_raises(self, bad):
from artemis.config import checker_overrides_for_level
with pytest.raises(ValueError, match="Unknown verification level"):
checker_overrides_for_level(bad)
def test_level_preset_survives_explicit_master_switch(self):
"""``--verification-level strict --disable-checker`` keeps the master switch off."""
from artemis.config import checker_overrides_for_level
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
builder = AgentConfigBuilder()
builder.with_checker(**checker_overrides_for_level("strict"))
assert builder._disable_checker is False
assert builder._disable_midway_checks is False
assert builder._assert_failure_policy == "halt"
builder.with_checker(enabled=False)
assert builder._disable_checker is True
# The preset's finer-grained fields survive the master switch flip.
assert builder._assert_failure_policy == "halt"
@pytest.mark.parametrize("level", ["off", "final", "checkpoints", "strict"])
def test_effective_config_classifies_back_onto_the_ladder(self, level):
from artemis.config import (
CheckerConfig,
checker_overrides_for_level,
verification_level_for_checker,
)
cfg = CheckerConfig(**checker_overrides_for_level(level))
assert verification_level_for_checker(cfg) == level
# A config with both gates switched off is "off" even if enabled=True.
assert (
verification_level_for_checker(
CheckerConfig(enabled=True, midway_checks=False, final_check=False)
)
== "off"
)
tests/unit/test_drivers.py
+1 -1
View File
@@ -118,7 +118,7 @@ async def test_android_driver_preserves_elements_from_combined_screen_data():
"text": "Start",
"class": "android.widget.Button",
"bounds": "[10,20][100,80]",
}
},
]
mock_ui_client.get_screen_data.return_value = SimpleNamespace(
base64=_ONE_PIXEL_PNG,
tests/unit/test_install_paths.py
+85
View File
@@ -0,0 +1,85 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Regression tests for source-checkout versus installed-wheel state paths."""
from artemis.config import paths
from artemis.config import runtime
def _clear_path_overrides(monkeypatch):
for name in (
"ARTEMIS_APP_DIR",
"ANTIGRAVITY_APP_DIR",
"ARTEMIS_TRACES_DIR",
"ARTEMIS_USE_USER_DIR",
):
monkeypatch.delenv(name, raising=False)
def test_installed_package_keeps_mutable_state_out_of_site_packages(tmp_path, monkeypatch):
_clear_path_overrides(monkeypatch)
installed_root = tmp_path / "site-packages"
(installed_root / "artemis").mkdir(parents=True)
user_app_dir = tmp_path / "user-data" / "artemis"
monkeypatch.setattr(paths, "ROOT_DIR", installed_root)
monkeypatch.setattr(paths.platform.paths, "resolve_app_dir", lambda: user_app_dir)
assert not paths.is_source_checkout()
assert paths.get_env_file() == user_app_dir / ".env"
assert paths.get_default_traces_path() == user_app_dir / "traces"
assert paths.get_ipc_port_file().parent == user_app_dir
assert paths.get_ls_address_file().parent == user_app_dir
assert paths.get_server_info_file().parent == user_app_dir
assert paths.get_pause_file().parent == user_app_dir
def test_source_checkout_preserves_workspace_defaults(tmp_path, monkeypatch):
_clear_path_overrides(monkeypatch)
source_root = tmp_path / "checkout"
(source_root / "artemis").mkdir(parents=True)
(source_root / "pyproject.toml").write_text("[project]\nname='artemis'\n", encoding="utf-8")
monkeypatch.setattr(paths, "ROOT_DIR", source_root)
# A developer machine may carry a real user-level port file; the source
# checkout contract must not depend on that ambient state.
monkeypatch.setattr(
paths.platform.paths, "resolve_app_dir", lambda: tmp_path / "user-data" / "artemis"
)
assert paths.is_source_checkout()
assert paths.get_env_file() == source_root / ".env"
assert paths.get_default_traces_path() == source_root / "traces"
assert paths.get_ipc_port_file().parent == source_root
assert paths.get_pause_file().parent == source_root
def test_installed_ipc_never_writes_site_packages_legacy_file(tmp_path, monkeypatch):
_clear_path_overrides(monkeypatch)
installed_root = tmp_path / "site-packages"
(installed_root / "artemis").mkdir(parents=True)
user_app_dir = tmp_path / "user-data" / "artemis"
temp_dir = tmp_path / "temp"
monkeypatch.setattr(paths, "ROOT_DIR", installed_root)
monkeypatch.setattr(runtime, "ROOT_DIR", installed_root)
monkeypatch.setattr(paths.platform.paths, "resolve_app_dir", lambda: user_app_dir)
monkeypatch.setattr(runtime, "get_temp_dir", lambda: temp_dir)
runtime.write_ipc_port(43123)
assert (user_app_dir / ".artemis_ipc_port").read_text(encoding="utf-8") == "43123"
assert not (installed_root / ".artemis_ipc_port").exists()
runtime.clear_ipc_port()
tests/unit/test_llm_reliability.py
+1 -3
View File
@@ -75,9 +75,7 @@ def test_retry_policy_accepts_extension_category_values():
RATE_LIMIT = "rate_limit"
MEDIA_PROCESSING = "media_processing"
assert retry_policy_for(ExtCategory.RATE_LIMIT) is retry_policy_for(
FailureCategory.RATE_LIMIT
)
assert retry_policy_for(ExtCategory.RATE_LIMIT) is retry_policy_for(FailureCategory.RATE_LIMIT)
assert retry_policy_for("provider_unavailable") is retry_policy_for(
FailureCategory.PROVIDER_UNAVAILABLE
)
tests/unit/test_memory_config.py
+3 -9
View File
@@ -33,9 +33,7 @@ def test_memory_defaults():
def test_memory_transcript_flag_explicit_disable_is_rollback_switch():
cfg = AgentGlobalConfig.model_validate(
{"memory": {"transcript": {"enabled": False}}}
)
cfg = AgentGlobalConfig.model_validate({"memory": {"transcript": {"enabled": False}}})
assert cfg.memory.transcript.enabled is False
# Sibling scrub defaults are unaffected by flipping the flag.
assert cfg.memory.transcript.image_scrub_depth == 3
@@ -64,9 +62,7 @@ def test_memory_explicit_values():
def test_legacy_step_summarizer_retry_limit_seeds_memory_runtime():
"""An explicit legacy flash.step_summarizer.retry_limit keeps working."""
cfg = AgentGlobalConfig.model_validate(
{"flash": {"step_summarizer": {"retry_limit": 7}}}
)
cfg = AgentGlobalConfig.model_validate({"flash": {"step_summarizer": {"retry_limit": 7}}})
assert cfg.flash.step_summarizer.retry_limit == 7
assert cfg.memory.runtime.retry_limit == 7
@@ -85,9 +81,7 @@ def test_explicit_memory_runtime_wins_over_legacy_alias():
def test_legacy_default_does_not_override_memory_default():
"""A step_summarizer block without retry_limit must not seed anything."""
cfg = AgentGlobalConfig.model_validate(
{"flash": {"step_summarizer": {"enabled": True}}}
)
cfg = AgentGlobalConfig.model_validate({"flash": {"step_summarizer": {"enabled": True}}})
assert cfg.memory.runtime.retry_limit == 3
tests/unit/test_packaging_resources.py
+56
View File
@@ -0,0 +1,56 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contract tests for release metadata and bundled runtime resources."""
from importlib.metadata import version
import artemis
from artemis import resources
def test_public_version_comes_from_distribution_metadata():
assert artemis.__version__ == version("artemis")
def test_source_tree_contains_complete_release_resources():
config_path = resources.get_bundled_config_path("artemis.jsonc")
showcase_path = resources.get_bundled_showcase_dist()
assert config_path is not None and config_path.is_file()
assert showcase_path is not None and (showcase_path / "index.html").is_file()
def test_bundled_resource_accessors_require_complete_assets(tmp_path, monkeypatch):
resource_root = tmp_path / "resources"
config_dir = resource_root / "config"
showcase_dir = resource_root / "showcase_ui"
config_dir.mkdir(parents=True)
showcase_dir.mkdir()
(config_dir / "artemis.jsonc").write_text("{}", encoding="utf-8")
(showcase_dir / "index.html").write_text("<html></html>", encoding="utf-8")
monkeypatch.setattr(resources, "files", lambda _package: resource_root)
assert resources.get_bundled_config_path("artemis.jsonc") == (config_dir / "artemis.jsonc")
assert resources.get_bundled_config_path("missing.jsonc") is None
assert resources.get_bundled_showcase_dist() == showcase_dir
def test_incomplete_showcase_resource_is_rejected(tmp_path, monkeypatch):
resource_root = tmp_path / "resources"
(resource_root / "showcase_ui").mkdir(parents=True)
monkeypatch.setattr(resources, "files", lambda _package: resource_root)
assert resources.get_bundled_showcase_dist() is None
tests/unit/test_planner.py
+17 -1
View File
@@ -47,7 +47,7 @@ async def test_planner_validation():
)
mock_llm.with_structured_output.return_value = mock_structured
with patch("artemis.agents.planner.planner.get_llm", return_value=mock_llm):
with patch("artemis.agents.planner.planner.get_llm", return_value=mock_llm) as get_llm:
res = await run_async_planner_validation(
ctx=DummyCtx(),
initial_goal="Do the task",
@@ -58,6 +58,22 @@ async def test_planner_validation():
)
assert res is not None
assert res.get("status") == "success"
# The review runs on the lightweight judge node, never the Planner model.
assert get_llm.call_args.kwargs.get("name") == "planner_validation"
def test_planner_validation_node_defaults_to_lightweight_judge():
"""Unconfigured planner_validation resolves to the same flash-lite default
as the pixel safety net (cheap, temperature 0)."""
from artemis.config import get_default_llm_config
llm_cfg = get_default_llm_config()
node = llm_cfg.get_agent("planner_validation")
safety_net = llm_cfg.get_agent("validator_pixel_safety_net")
assert node.model == safety_net.model
assert node.provider == safety_net.provider
assert node.temperature == 0.0
assert "lite" in node.model
if __name__ == "__main__":
tests/unit/test_token_meter.py
+1 -3
View File
@@ -75,9 +75,7 @@ def test_extract_usage_absent_returns_none():
def test_meter_accumulates_session_totals():
meter = SessionTokenMeter("s1")
meter.record({"prompt_tokens": 100, "completion_tokens": 10, "cached_tokens": 0})
snapshot = meter.record(
{"prompt_tokens": 300, "completion_tokens": 20, "cached_tokens": 250}
)
snapshot = meter.record({"prompt_tokens": 300, "completion_tokens": 20, "cached_tokens": 250})
assert snapshot == {
"session_llm_calls": 2,
"session_prompt_tokens": 400,
tests/unit/test_unified_controller_video.py
+9 -6
View File
@@ -271,12 +271,15 @@ async def test_unified_controller_stop_recording(mock_ctx, tmp_path):
)
set_active_session("emulator-5554", session)
with patch(
"artemis.controllers.unified_controller.remux_recording_to_mp4",
AsyncMock(side_effect=lambda src, dst: (dst.write_bytes(b"mp4 content"), True)[1]),
), patch(
"artemis.controllers.unified_controller.write_recording_manifest",
AsyncMock(return_value=tmp_path / "recording.json"),
with (
patch(
"artemis.controllers.unified_controller.remux_recording_to_mp4",
AsyncMock(side_effect=lambda src, dst: (dst.write_bytes(b"mp4 content"), True)[1]),
),
patch(
"artemis.controllers.unified_controller.write_recording_manifest",
AsyncMock(return_value=tmp_path / "recording.json"),
),
):
res = await controller.stop_video_recording()
tests/unit/tools/test_command_tool.py
+168 -15
View File
@@ -16,8 +16,7 @@ import asyncio
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.context import ArtemisContext
from artemis.tools.command_tool import (
_BACKGROUND_TASKS,
_PERSISTENT_ENVIRONMENTS,
get_adb_task_registry,
get_manage_task_tool,
get_run_adb_command_tool,
get_run_short_adb_command_tool,
@@ -120,7 +119,7 @@ async def test_run_command_sync(mock_exec, mock_ctx):
mock_exec.assert_called_once()
# Check that adb was called with device ID and "shell"
args, kwargs = mock_exec.call_args
assert args[0] == "adb"
assert args[0].lower().replace(".exe", "").endswith("adb")
assert args[1] == "-s"
assert args[2] == "test_device_1234"
assert args[3] == "shell"
@@ -149,8 +148,9 @@ async def test_run_command_persistent_env(mock_exec, mock_ctx):
}
)
assert "completed with exit code 0" in res1
assert terminal_id in _PERSISTENT_ENVIRONMENTS
assert _PERSISTENT_ENVIRONMENTS[terminal_id].get("ARTEMIS_TEST_ENV") == "coffee"
envs = get_adb_task_registry(mock_ctx).persistent_envs
assert terminal_id in envs
assert envs[terminal_id].get("ARTEMIS_TEST_ENV") == "coffee"
# 2. Query environment variable in subsequent command on same terminal
mock_process_2 = MockProcess(
@@ -186,7 +186,8 @@ async def test_run_command_async_and_status(mock_exec, mock_ctx):
run_command = get_run_adb_command_tool(mock_ctx)
manage_task = get_manage_task_tool(mock_ctx)
_BACKGROUND_TASKS.clear()
registry = get_adb_task_registry(mock_ctx)
registry.background.clear()
res = await run_command.ainvoke(
{
@@ -204,7 +205,7 @@ async def test_run_command_async_and_status(mock_exec, mock_ctx):
task_id_line = [line for line in lines if line.startswith("TaskId: ")][0]
task_id = task_id_line.split(": ")[1]
assert task_id in _BACKGROUND_TASKS
assert task_id in registry.background
# Check status (should be running)
status_res = await manage_task.ainvoke(
@@ -219,7 +220,7 @@ async def test_run_command_async_and_status(mock_exec, mock_ctx):
await asyncio.sleep(1.2)
# Status should be completed (and task removed from active map)
assert task_id not in _BACKGROUND_TASKS
assert task_id not in registry.background
@pytest.mark.asyncio
@@ -231,7 +232,8 @@ async def test_run_command_kill(mock_exec, mock_ctx):
run_command = get_run_adb_command_tool(mock_ctx)
manage_task = get_manage_task_tool(mock_ctx)
_BACKGROUND_TASKS.clear()
registry = get_adb_task_registry(mock_ctx)
registry.background.clear()
res = await run_command.ainvoke(
{
@@ -254,7 +256,7 @@ async def test_run_command_kill(mock_exec, mock_ctx):
}
)
assert f"Task {task_id} successfully terminated" in kill_res
assert task_id not in _BACKGROUND_TASKS
assert task_id not in registry.background
@pytest.mark.asyncio
@@ -275,7 +277,7 @@ async def test_run_short_command_success(mock_exec, mock_ctx):
assert "short command success output" in result
mock_exec.assert_called_once()
args, kwargs = mock_exec.call_args
assert args[0] == "adb"
assert args[0].lower().replace(".exe", "").endswith("adb")
assert args[1] == "-s"
assert args[2] == "test_device_1234"
assert args[3] == "shell"
@@ -453,12 +455,12 @@ def test_manage_task_tool_subclass_and_registry():
async def test_manage_task_tool_direct_execute(mock_ctx):
"""Verify direct ManageTaskTool.execute execution."""
from artemis.tools.command_tool import (
_BACKGROUND_TASKS,
BackgroundTask,
manage_task,
)
_BACKGROUND_TASKS.clear()
registry = get_adb_task_registry(mock_ctx)
registry.background.clear()
list_res = await manage_task.execute(ctx=mock_ctx, Action="list")
assert "No active background tasks" in list_res
@@ -471,7 +473,7 @@ async def test_manage_task_tool_direct_execute(mock_ctx):
terminal_id="term_1",
cwd="/data/local/tmp",
)
_BACKGROUND_TASKS["dummy_task_99"] = bg_task
registry.background["dummy_task_99"] = bg_task
list_res2 = await manage_task.execute(ctx=mock_ctx, Action="list")
assert "dummy_task_99" in list_res2
@@ -482,7 +484,8 @@ async def test_manage_task_tool_direct_execute(mock_ctx):
assert "Status: running" in status_res
assert "dummy_task_99" in status_res
_BACKGROUND_TASKS.clear()
registry = get_adb_task_registry(mock_ctx)
registry.background.clear()
def test_analyze_task_output_tool_subclass_and_registry():
@@ -601,3 +604,153 @@ async def test_run_short_adb_command_tool_mock_driver_execute():
)
assert "mock_output: getprop ro.build.version.release" in result
assert "getprop ro.build.version.release" in driver.action_history[-1]["command"]
# ---------------------------------------------------------------------------
# Registry, exit codes, stdin control, cloud timeout, shutdown
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
@patch("asyncio.create_subprocess_exec")
async def test_persistent_reports_script_exit_code(mock_exec, mock_ctx):
"""A failing command in a persistent terminal must not report exit code 0."""
mock_exec.return_value = MockProcess(
output_bytes=b"boom\n===EXIT_CODE===3\n===ENV_START===\nPATH=/bin\nFOO=bar baz\n",
exit_code=0, # the trailing `env` succeeds, the command did not
)
run_command = get_run_adb_command_tool(mock_ctx)
res = await run_command.ainvoke(
{
"CommandLine": "false",
"RunPersistent": True,
"RequestedTerminalID": "term_exit",
"WaitMsBeforeAsync": 500,
}
)
assert "completed with exit code 3" in res
assert "===EXIT_CODE===" not in res
envs = get_adb_task_registry(mock_ctx).persistent_envs["term_exit"]
assert envs == {"FOO": "bar baz"} # platform vars like PATH are dropped
def test_build_phone_script_quotes_env_and_cwd():
import shlex
from artemis.tools.command_tool import _build_phone_script
script = _build_phone_script("echo hi", "/sdcard/my dir", {"FOO": "it's"}, True)
assert script.startswith("cd '/sdcard/my dir'\n")
expected_export = "export FOO=" + shlex.quote("it's") + "\n"
assert expected_export in script
assert "===EXIT_CODE===$_artemis_ec" in script
assert script.rstrip().endswith("env")
@pytest.mark.asyncio
@patch("asyncio.create_subprocess_exec")
async def test_stdin_closed_unless_interactive(mock_exec, mock_ctx):
mock_exec.return_value = MockProcess(output_bytes=b"ok\n", exit_code=0)
run_command = get_run_adb_command_tool(mock_ctx)
await run_command.ainvoke({"CommandLine": "cat", "WaitMsBeforeAsync": 500})
assert mock_exec.call_args.kwargs["stdin"] == asyncio.subprocess.DEVNULL
mock_exec.return_value = MockProcess(output_bytes=b"ok\n", exit_code=0)
await run_command.ainvoke({"CommandLine": "cat", "Interactive": True, "WaitMsBeforeAsync": 500})
assert mock_exec.call_args.kwargs["stdin"] == asyncio.subprocess.PIPE
@pytest.mark.asyncio
@patch("asyncio.create_subprocess_exec")
async def test_send_input_rejected_for_non_interactive_task(mock_exec, mock_ctx):
mock_exec.return_value = MockProcess(output_bytes=b"", exit_code=0, delay=5.0)
run_command = get_run_adb_command_tool(mock_ctx)
manage_task = get_manage_task_tool(mock_ctx)
registry = get_adb_task_registry(mock_ctx)
res = await run_command.ainvoke({"CommandLine": "sleep 10", "WaitMsBeforeAsync": 100})
task_id = [ln for ln in res.splitlines() if ln.startswith("TaskId: ")][0].split(": ")[1]
assert "stdin is closed" in res
reply = await manage_task.ainvoke({"Action": "send_input", "TaskId": task_id, "Input": "y\n"})
assert "Interactive=true" in reply
await manage_task.ainvoke({"Action": "kill", "TaskId": task_id})
assert task_id not in registry.background
@pytest.mark.asyncio
@patch("asyncio.create_subprocess_exec")
async def test_shutdown_kills_background_tasks_and_records_them(mock_exec, mock_ctx):
from artemis.tools.command_tool import shutdown_adb_background_tasks
mock_exec.return_value = MockProcess(output_bytes=b"", exit_code=0, delay=5.0)
run_command = get_run_adb_command_tool(mock_ctx)
registry = get_adb_task_registry(mock_ctx)
res = await run_command.ainvoke({"CommandLine": "logcat", "WaitMsBeforeAsync": 100})
task_id = [ln for ln in res.splitlines() if ln.startswith("TaskId: ")][0].split(": ")[1]
assert task_id in registry.background
killed = await shutdown_adb_background_tasks(mock_ctx)
assert killed == 1
assert registry.background == {}
assert registry.finished[task_id]["status"] == "killed"
# A second shutdown is a no-op.
assert await shutdown_adb_background_tasks(mock_ctx) == 0
# A context that never used ADB tools has no registry and is a no-op too.
fresh = MagicMock(spec=ArtemisContext)
assert await shutdown_adb_background_tasks(fresh) == 0
def test_registry_is_per_context_and_notifications_do_not_leak():
from artemis.tools.command_tool import _register_finished_task
ctx_a = MagicMock(spec=ArtemisContext)
ctx_b = MagicMock(spec=ArtemisContext)
reg_a = get_adb_task_registry(ctx_a)
reg_b = get_adb_task_registry(ctx_b)
assert reg_a is not reg_b
assert get_adb_task_registry(ctx_a) is reg_a
_register_finished_task("t_a", "logcat -d", None, None, "completed", 0, "out", ctx=ctx_a)
assert [t["task_id"] for t in reg_a.pop_unnotified_finished()] == ["t_a"]
assert reg_b.pop_unnotified_finished() == []
# Consumed once: the next Operator turn on ctx_a does not see it again.
assert reg_a.pop_unnotified_finished() == []
@pytest.mark.asyncio
@patch("asyncio.create_subprocess_exec")
async def test_sync_long_output_is_not_reannounced(mock_exec, mock_ctx):
long_output = "\n".join(f"line {i}" for i in range(600))
mock_exec.return_value = MockProcess(output_bytes=long_output.encode(), exit_code=0)
run_command = get_run_adb_command_tool(mock_ctx)
registry = get_adb_task_registry(mock_ctx)
registry.finished.clear()
res = await run_command.ainvoke({"CommandLine": "dumpsys", "WaitMsBeforeAsync": 500})
assert "has been truncated" in res
assert registry.pop_unnotified_finished() == []
# ... but analyze_task_output can still reach the full text.
task_id = res.split("TaskId: ")[1].split(".")[0]
assert "line 0" in registry.get_task_info(task_id)["output"]
@pytest.mark.asyncio
async def test_cloud_mode_command_has_hard_timeout(mock_ctx, monkeypatch):
import time
monkeypatch.setenv("ARTEMIS_CLOUD_MODE", "1")
device = MagicMock()
device.shell = MagicMock(side_effect=lambda _script: time.sleep(1.0) or "late")
with patch("artemis.tools.command_tool.get_adb_device", return_value=device):
run_command = get_run_adb_command_tool(mock_ctx)
res = await run_command.ainvoke({"CommandLine": "logcat", "WaitMsBeforeAsync": 100})
assert "did not finish within" in res
device.shell = MagicMock(return_value="fast\n")
with patch("artemis.tools.command_tool.get_adb_device", return_value=device):
res = await run_command.ainvoke({"CommandLine": "echo fast", "WaitMsBeforeAsync": 500})
assert "fast" in res
tests/unit/tools/test_explorer_tool.py
+580 -311
View File
@@ -12,364 +12,633 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from unittest.mock import AsyncMock, MagicMock, mock_open, patch
"""``ask_explorer``: tier-agnostic contract and the locate / register / render pipeline."""
import json
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from artemis.agents.explorer.constants import (
ASK_EXPLORER_CONTEXT_FEEDBACK_DESCRIPTION,
ASK_EXPLORER_DESCRIPTION,
ASK_EXPLORER_QUERY_DESCRIPTION,
)
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.explorer_tool import (
AskExplorer,
AskExplorerArgs,
AskExplorerTool,
AskExplorerToolAlias,
ExplorerCandidate,
ExplorerOutcome,
RegisteredCandidate,
_run_explorer_logic,
ask_explorer,
ask_explorer_text,
ask_explorer_wrapper,
get_ask_explorer_tool,
locate,
register_candidates,
render_operator_blocks,
render_text,
)
import pytest
# --------------------------------------------------------------------------- #
# Fixtures
# --------------------------------------------------------------------------- #
def test_ask_explorer_tool_subclass_and_registry():
"""Verify AskExplorerTool is a subclass of ArtemisTool and properly registered."""
@pytest.fixture(autouse=True)
def _no_env_tier(monkeypatch):
"""The environment override must not leak into tier-resolution assertions."""
monkeypatch.delenv("ARTEMIS_EXPLORER_VERSION", raising=False)
def _ctx(width: int | None = 1080, height: int | None = 2400, base_dir: Path | None = None):
ctx = MagicMock(spec=ArtemisContext)
ctx.agent_config = None
ctx.llm_config = None
ctx.execution_setup = None
if base_dir is None:
ctx.data_engine = None
else:
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(base_dir)
if width is None:
ctx.device = None
else:
ctx.device = MagicMock()
ctx.device.device_width = width
ctx.device.device_height = height
return ctx
def _state(screenshot: str | None = "/tmp/test.jpg", raw: dict | None = None):
state = MagicMock(spec=State)
state.latest_screenshot = screenshot
state.indexed_points = []
state.indexed_elements = []
state.operator_raw_data = raw
return state
def _explorer_returning(raw):
instance = MagicMock()
instance.run = AsyncMock(return_value=raw)
return patch("artemis.tools.explorer_tool.Explorer", return_value=instance), instance
# --------------------------------------------------------------------------- #
# Contract
# --------------------------------------------------------------------------- #
def test_tool_contract_is_tier_agnostic():
assert issubclass(AskExplorerTool, ArtemisTool)
assert issubclass(AskExplorer, ArtemisTool)
assert issubclass(AskExplorerToolAlias, ArtemisTool)
assert isinstance(ask_explorer, ArtemisTool)
assert isinstance(ask_explorer, AskExplorerTool)
assert ask_explorer.name == "ask_explorer"
assert ask_explorer.category == "explorer"
assert ask_explorer.args_schema == AskExplorerArgs
assert ask_explorer.args_schema is AskExplorerArgs
# Registry lookup
reg_tool = ToolRegistry.get("ask_explorer")
assert reg_tool is not None
assert isinstance(reg_tool, AskExplorerTool)
fields = AskExplorerArgs.model_fields
assert set(fields) == {"query", "context_feedback"}
assert fields["query"].is_required()
assert fields["query"].description == ASK_EXPLORER_QUERY_DESCRIPTION
assert fields["context_feedback"].default == ""
assert fields["context_feedback"].description == ASK_EXPLORER_CONTEXT_FEEDBACK_DESCRIPTION
# GenAI FunctionDeclaration export
declaration = ask_explorer.to_genai_declaration()
assert declaration.name == "ask_explorer"
assert "query" in declaration.parameters.properties
# Wrapper check
assert ask_explorer_wrapper is not None
assert ask_explorer_wrapper.tool_fn_getter == get_ask_explorer_tool
assert set(declaration.parameters.properties) == {"query", "context_feedback"}
assert declaration.parameters.required == ["query"]
@pytest.mark.asyncio
async def test_ask_explorer_multiple_candidates():
# Mock Context and State
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.agent_config = None
mock_ctx.llm_config = None
mock_ctx.execution_setup = None
mock_ctx.device = MagicMock()
mock_ctx.device.device_width = 1080
mock_ctx.device.device_height = 2400
@pytest.mark.parametrize("version", [None, "flash", "pro", "ultra"])
def test_description_does_not_depend_on_the_tier(version):
tool = AskExplorerTool(version=version)
assert tool.description == ASK_EXPLORER_DESCRIPTION
assert tool.version == version
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test.jpg"
mock_state.indexed_points = [[100, 200]] # Pre-existing point
mock_state.indexed_elements = [
def test_registry_and_wrapper_export():
reg_tool = ToolRegistry.get("ask_explorer")
assert isinstance(reg_tool, AskExplorerTool)
assert ask_explorer_wrapper.tool_fn_getter is get_ask_explorer_tool
assert ask_explorer_wrapper.on_success_fn("x") == "x"
assert "boom" in ask_explorer_wrapper.on_failure_fn("boom")
# --------------------------------------------------------------------------- #
# ExplorerOutcome.from_raw
# --------------------------------------------------------------------------- #
def test_from_raw_parses_candidates_and_message():
raw = json.dumps(
{
"index": 1,
"center": [100, 200],
"text": "Pre-existing",
"bounds": None,
"class": "Test",
"resource_id": None,
"is_ocr": False,
"candidates": [
{"label": "S1", "coords": [500, 500], "description": "First Button"},
{"label": "S2", "coords": [250, 750], "description": "Second Button"},
],
"fallback_message": "Please choose carefully.",
}
)
outcome = ExplorerOutcome.from_raw(raw)
assert outcome.found and not outcome.error
assert outcome.candidates == [
ExplorerCandidate(label="S1", coords=(500, 500), description="First Button"),
ExplorerCandidate(label="S2", coords=(250, 750), description="Second Button"),
]
assert outcome.message == "Please choose carefully."
assert outcome.raw == raw
# Mock Explorer Agent Outcome with multiple candidates and fallback message
explorer_outcome = {
"candidates": [
{
"label": "S1",
"coords": [500, 500],
"description": "First Button",
},
{
"label": "S2",
"coords": [250, 750],
"description": "Second Button",
},
],
"fallback_message": "Please choose carefully.",
}
explorer_output_str = json.dumps(explorer_outcome)
# We need to patch the Explorer class inside the tool
mock_explorer_instance = MagicMock()
mock_explorer_instance.run = AsyncMock(return_value=explorer_output_str)
def test_from_raw_accepts_a_dict_and_fills_defaults():
outcome = ExplorerOutcome.from_raw(
{"candidates": [{"coords": ["10", "20"]}]}, default_label="gear icon"
)
assert len(outcome.candidates) == 1
cand = outcome.candidates[0]
assert cand.label == "gear icon"
assert cand.description == "gear icon"
assert cand.coords == (10, 20)
assert outcome.message == ""
with patch(
"artemis.tools.explorer_tool.Explorer",
return_value=mock_explorer_instance,
):
# Retrieve the tool
ask_explorer_tool = get_ask_explorer_tool(mock_ctx, version="pro")
result = await ask_explorer_tool.ainvoke(
{
"query": "Find buttons",
"context_feedback": "Attempt 1",
"state": mock_state,
}
)
def test_from_raw_drops_candidates_with_invalid_coords():
outcome = ExplorerOutcome.from_raw(
{
"candidates": [
{"label": "bad1", "coords": [1500, 10]},
{"label": "bad2", "coords": [10]},
{"label": "bad3", "coords": "10,10"},
{"label": "bad4"},
"not a dict",
{"label": "ok", "coords": [10, 10]},
]
}
)
assert [c.label for c in outcome.candidates] == ["ok"]
assert not outcome.error
assert mock_state.indexed_points == [
[100, 200],
[540, 1200],
[270, 1800],
def test_from_raw_unreadable_text_is_an_error_outcome():
outcome = ExplorerOutcome.from_raw("I could not parse the screen {")
assert outcome.error and not outcome.found
assert "unreadable answer" in outcome.message
assert outcome.raw == "I could not parse the screen {"
def test_from_raw_non_object_json_is_an_error_outcome():
outcome = ExplorerOutcome.from_raw("[1, 2, 3]")
assert outcome.error
assert "unexpected answer" in outcome.message
def test_from_raw_empty_candidates_is_a_clean_not_found():
outcome = ExplorerOutcome.from_raw({"candidates": [], "fallback_message": "Covered."})
assert not outcome.found and not outcome.error
assert outcome.message == "Covered."
# --------------------------------------------------------------------------- #
# register_candidates
# --------------------------------------------------------------------------- #
def test_register_candidates_appends_after_existing_elements_using_observation_size():
ctx = _ctx()
state = _state(raw={"width": 1000, "height": 2000})
state.indexed_points = [[100, 200]]
state.indexed_elements = [{"index": 1, "center": [100, 200], "text": "Pre-existing"}]
outcome = ExplorerOutcome(
candidates=[
ExplorerCandidate(label="S1", coords=(500, 500), description="First Button"),
ExplorerCandidate(label="S2", coords=(250, 750), description="Second Button"),
]
)
assert len(mock_state.indexed_elements) == 3
assert mock_state.indexed_elements[1]["index"] == 2
assert mock_state.indexed_elements[1]["center"] == [540, 1200]
assert mock_state.indexed_elements[1]["text"] == "First Button"
assert mock_state.indexed_elements[2]["index"] == 3
assert mock_state.indexed_elements[2]["center"] == [270, 1800]
assert mock_state.indexed_elements[2]["text"] == "Second Button"
registered = register_candidates(ctx, state, outcome)
assert isinstance(result, str)
assert "Explorer successfully located the following candidate(s):" in result
assert "- [2] 'First Button' at coordinate [500, 500]" in result
assert "- [3] 'Second Button' at coordinate [250, 750]" in result
assert (
"You can click/act on them directly by calling perform_action with"
" their respective index." in result
)
assert "Additional Notes from Explorer: Please choose carefully." in result
@pytest.mark.asyncio
async def test_ask_explorer_only_fallback_message():
# Mock Context and State
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.agent_config = None
mock_ctx.llm_config = None
mock_ctx.execution_setup = None
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test.jpg"
mock_state.indexed_points = []
mock_state.indexed_elements = []
explorer_outcome = {
"candidates": [],
"fallback_message": ("Element not found because it is covered by a keyboard."),
}
explorer_output_str = json.dumps(explorer_outcome)
mock_explorer_instance = MagicMock()
mock_explorer_instance.run = AsyncMock(return_value=explorer_output_str)
with patch(
"artemis.tools.explorer_tool.Explorer",
return_value=mock_explorer_instance,
):
ask_explorer_tool = get_ask_explorer_tool(mock_ctx, version="pro")
result = await ask_explorer_tool.ainvoke(
{
"query": "Find hidden button",
"context_feedback": "",
"state": mock_state,
}
)
assert mock_state.indexed_points == []
assert mock_state.indexed_elements == []
assert (
"Explorer could not locate the element. Message: Element not found"
" because it is covered by a keyboard." in result
)
@pytest.mark.asyncio
async def test_ask_explorer_multimodal_success():
# Mock Context and State
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.agent_config = None
mock_ctx.llm_config = None
mock_ctx.execution_setup = None
mock_ctx.device = MagicMock()
mock_ctx.device.device_width = 1080
mock_ctx.device.device_height = 2400
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test.jpg"
mock_state.indexed_points = [[100, 200]]
mock_state.indexed_elements = [
{
"index": 1,
"center": [100, 200],
"text": "Pre-existing",
"bounds": None,
"class": "Test",
"resource_id": None,
"is_ocr": False,
}
assert registered == [
RegisteredCandidate(
index=2, pixel=(500, 1000), coords=(500, 500), description="First Button"
),
RegisteredCandidate(
index=3, pixel=(250, 1500), coords=(250, 750), description="Second Button"
),
]
explorer_outcome = {
"candidates": [
{
"label": "S1",
"coords": [500, 500],
"description": "First Button",
},
],
"fallback_message": "",
assert state.indexed_points == [[100, 200], [500, 1000], [250, 1500]]
assert len(state.indexed_elements) == 3
assert state.indexed_elements[1] == {
"index": 2,
"center": [500, 1000],
"text": "First Button",
"bounds": None,
"class": "ExplorerCandidate",
"resource_id": None,
"is_ocr": False,
}
explorer_output_str = json.dumps(explorer_outcome)
mock_explorer_instance = MagicMock()
mock_explorer_instance.run = AsyncMock(return_value=explorer_output_str)
m_open = mock_open(read_data=b"fake_annotated_image_bytes")
with (
patch(
"artemis.tools.explorer_tool.Explorer",
return_value=mock_explorer_instance,
),
patch("artemis.tools.explorer_tool.draw_dots") as mock_draw_dots,
patch("builtins.open", m_open),
patch("pathlib.Path.mkdir"),
):
ask_explorer_tool = get_ask_explorer_tool(mock_ctx, version="pro")
result = await ask_explorer_tool.ainvoke(
{
"query": "Find buttons",
"context_feedback": "",
"state": mock_state,
}
)
assert isinstance(result, list)
assert len(result) == 2
assert result[0]["type"] == "text"
assert "Explorer successfully located" in result[0]["text"]
assert result[1]["type"] == "image_url"
assert result[1]["image_url"]["url"].startswith("data:image/jpeg;base64,")
mock_draw_dots.assert_called_once()
points_arg = mock_draw_dots.call_args[0][1]
assert points_arg == [[100, 200], [540, 1200]]
labels_arg = mock_draw_dots.call_args[0][2]
assert labels_arg == ["1", "2"]
assert state.indexed_elements[2]["index"] == 3
assert state.indexed_elements[2]["center"] == [250, 1500]
@pytest.mark.asyncio
async def test_ask_explorer_direct_execution():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.agent_config = None
mock_ctx.llm_config = None
mock_ctx.execution_setup = None
mock_ctx.device = None
def test_register_candidates_falls_back_to_the_device_size():
ctx = _ctx(1080, 2400)
state = _state(raw=None)
outcome = ExplorerOutcome(candidates=[ExplorerCandidate("S1", (500, 500), "Btn")])
registered = register_candidates(ctx, state, outcome)
assert registered[0].pixel == (540, 1200)
assert registered[0].index == 1
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test.jpg"
mock_state.indexed_points = []
mock_state.indexed_elements = []
mock_explorer_instance = MagicMock()
mock_explorer_instance.run = AsyncMock(
return_value=json.dumps(
{
"candidates": [],
"fallback_message": "Direct execution not found.",
}
)
def test_register_candidates_creates_the_lists_when_the_state_has_none():
ctx = _ctx()
state = MagicMock(spec=State)
state.indexed_points = None
state.indexed_elements = None
state.operator_raw_data = None
outcome = ExplorerOutcome(candidates=[ExplorerCandidate("S1", (0, 0), "Origin")])
registered = register_candidates(ctx, state, outcome)
assert registered[0].index == 1
assert state.indexed_points == [[0, 0]]
assert state.indexed_elements[0]["text"] == "Origin"
def test_register_candidates_is_a_no_op_without_candidates():
ctx = _ctx()
state = _state()
assert register_candidates(ctx, state, ExplorerOutcome(message="nope")) == []
assert register_candidates(ctx, state, ExplorerOutcome.failure("boom")) == []
assert state.indexed_points == []
assert state.indexed_elements == []
# --------------------------------------------------------------------------- #
# render_text / render_operator_blocks
# --------------------------------------------------------------------------- #
def test_render_text_found_lists_indices_and_coordinates():
outcome = ExplorerOutcome(
candidates=[ExplorerCandidate("S1", (500, 500), "First Button")],
message="Please choose carefully.",
)
registered = [RegisteredCandidate(2, (540, 1200), (500, 500), "First Button")]
text = render_text("Find buttons", outcome, registered)
assert "Explorer located 1 candidate(s) for 'Find buttons'" in text
assert "- [2] 'First Button' at normalized [500, 500]" in text
assert "target=[index]" in text
assert "Explorer notes: Please choose carefully." in text
with patch(
"artemis.tools.explorer_tool.Explorer",
return_value=mock_explorer_instance,
):
result = await ask_explorer.execute(
ctx=mock_ctx, state=mock_state, query="Find icon", version="pro"
)
assert "Direct execution not found." in result
def test_render_text_not_found_explains_and_suggests_rephrasing():
text = render_text("hidden button", ExplorerOutcome(message="Covered by keyboard."), [])
assert text.startswith("Explorer could not locate 'hidden button'. Covered by keyboard.")
assert "describing the target differently" in text
# Without an explanation the text still reads as a complete sentence.
text = render_text("x", ExplorerOutcome(), [])
assert "It gave no further detail." in text
def test_render_text_error_reports_the_failure():
text = render_text("gear icon", ExplorerOutcome.failure("Explorer failed: quota"), [])
assert text == "Explorer could not run for 'gear icon'. Explorer failed: quota"
def test_render_operator_blocks_adds_the_annotated_image(tmp_path):
ctx = _ctx(base_dir=tmp_path)
state = _state(screenshot=str(tmp_path / "shot.jpg"))
outcome = ExplorerOutcome(candidates=[ExplorerCandidate("S1", (500, 500), "First Button")])
registered = [RegisteredCandidate(2, (540, 1200), (500, 500), "First Button")]
def fake_draw_dots(screenshot_path, points, labels, output_path, **kwargs):
Path(output_path).write_bytes(b"fake_annotated_image_bytes")
with patch("artemis.tools.explorer_tool.draw_dots", side_effect=fake_draw_dots) as draw:
result = render_operator_blocks(ctx, state, "Find buttons", outcome, registered)
draw.assert_called_once()
args, kwargs = draw.call_args
assert args[0] == str(tmp_path / "shot.jpg")
assert args[1] == [[540, 1200]]
assert args[2] == ["2"]
assert Path(args[3]).parent == tmp_path / "images" / "explorer_tool"
assert kwargs["color"] == "magenta"
assert isinstance(result, list) and len(result) == 2
assert result[0]["type"] == "text"
assert "Explorer located 1 candidate(s)" in result[0]["text"]
assert result[1]["type"] == "image_url"
assert result[1]["image_url"]["url"].startswith("data:image/jpeg;base64,")
def test_render_operator_blocks_numbers_annotations_sequentially(tmp_path):
ctx = _ctx(base_dir=tmp_path)
state = _state(screenshot=str(tmp_path / "shot.jpg"))
outcome = ExplorerOutcome(candidates=[ExplorerCandidate("S1", (500, 500), "B")])
registered = [RegisteredCandidate(1, (540, 1200), (500, 500), "B")]
seen: list[str] = []
def fake_draw_dots(screenshot_path, points, labels, output_path, **kwargs):
seen.append(Path(output_path).name)
Path(output_path).write_bytes(b"img")
with patch("artemis.tools.explorer_tool.draw_dots", side_effect=fake_draw_dots):
render_operator_blocks(ctx, state, "B", outcome, registered)
render_operator_blocks(ctx, state, "B", outcome, registered)
assert seen == ["explorer_output_1.jpg", "explorer_output_2.jpg"]
def test_render_operator_blocks_degrades_to_text_when_drawing_fails(tmp_path):
ctx = _ctx(base_dir=tmp_path)
state = _state(screenshot=str(tmp_path / "shot.jpg"))
outcome = ExplorerOutcome(candidates=[ExplorerCandidate("S1", (500, 500), "First Button")])
registered = [RegisteredCandidate(2, (540, 1200), (500, 500), "First Button")]
with patch("artemis.tools.explorer_tool.draw_dots", side_effect=OSError("no PIL")):
result = render_operator_blocks(ctx, state, "Find buttons", outcome, registered)
assert isinstance(result, str)
assert "Explorer located 1 candidate(s)" in result
def test_render_operator_blocks_is_plain_text_without_candidates(tmp_path):
ctx = _ctx(base_dir=tmp_path)
state = _state(screenshot=str(tmp_path / "shot.jpg"))
with patch("artemis.tools.explorer_tool.draw_dots") as draw:
result = render_operator_blocks(ctx, state, "x", ExplorerOutcome(message="Nope."), [])
draw.assert_not_called()
assert isinstance(result, str)
assert "Explorer could not locate 'x'. Nope." in result
# --------------------------------------------------------------------------- #
# locate
# --------------------------------------------------------------------------- #
@pytest.mark.asyncio
async def test_ask_explorer_callable_execution():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.agent_config = None
mock_ctx.llm_config = None
mock_ctx.execution_setup = None
mock_ctx.device = None
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test.jpg"
mock_state.indexed_points = []
mock_state.indexed_elements = []
mock_explorer_instance = MagicMock()
mock_explorer_instance.run = AsyncMock(
return_value=json.dumps(
{
"candidates": [],
"fallback_message": "Callable test message.",
}
)
)
with patch(
"artemis.tools.explorer_tool.Explorer",
return_value=mock_explorer_instance,
):
result = await ask_explorer(
ctx=mock_ctx, state=mock_state, query="Search bar", version="pro"
)
assert "Callable test message." in result
@pytest.mark.asyncio
async def test_ask_explorer_flash_mode():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.agent_config = None
mock_ctx.llm_config = None
mock_ctx.execution_setup = None
mock_ctx.device = MagicMock()
mock_ctx.device.device_width = 1080
mock_ctx.device.device_height = 2400
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test.jpg"
mock_state.indexed_points = []
mock_state.indexed_elements = []
async def test_locate_runs_the_resolved_tier_and_parses_the_answer():
ctx = _ctx()
state = _state()
raw = json.dumps({"candidates": [{"label": "S1", "coords": [500, 500]}]})
explorer_patch, instance = _explorer_returning(raw)
with (
explorer_patch,
patch(
"artemis.tools.explorer_tool._run_object_detection",
new_callable=AsyncMock,
return_value={
"detected": [
{"label": "Icon", "point": [500, 500]},
],
"failed": [],
},
),
patch(
"builtins.open",
mock_open(read_data=json.dumps({"templates": [], "instructions": ""})),
),
"artemis.tools.explorer_tool.resolve_explorer_version", return_value="ultra"
) as resolve,
):
result = await ask_explorer.execute(
ctx=mock_ctx, state=mock_state, query="Icon", version="flash"
outcome = await locate(ctx, state, "gear icon", "was wrong", agent_name="validator")
resolve.assert_called_once_with(ctx, explicit_version=None, agent_or_profile_name="validator")
instance.run.assert_awaited_once_with(
"gear icon", "was wrong", "/tmp/test.jpg", state, version="ultra"
)
assert outcome.found and not outcome.error
assert outcome.candidates[0].coords == (500, 500)
@pytest.mark.asyncio
async def test_locate_forwards_a_programmatic_version_pin():
ctx = _ctx()
state = _state()
explorer_patch, instance = _explorer_returning(json.dumps({"candidates": []}))
with (
explorer_patch,
patch(
"artemis.tools.explorer_tool.resolve_explorer_version", return_value="pro"
) as resolve,
):
await locate(ctx, state, "q", version="pro", agent_name="flash")
resolve.assert_called_once_with(ctx, explicit_version="pro", agent_or_profile_name="flash")
assert instance.run.await_args.kwargs["version"] == "pro"
@pytest.mark.asyncio
async def test_locate_unknown_resolved_tier_falls_back_to_the_default_tier():
ctx = _ctx()
state = _state()
explorer_patch, instance = _explorer_returning(json.dumps({"candidates": []}))
with (
explorer_patch,
patch("artemis.tools.explorer_tool.resolve_explorer_version", return_value="turbo"),
):
await locate(ctx, state, "q")
assert instance.run.await_args.kwargs["version"] == "flash"
@pytest.mark.asyncio
async def test_locate_without_a_screenshot_is_an_error_outcome():
ctx = _ctx()
state = _state(screenshot=None)
with patch("artemis.tools.explorer_tool.Explorer") as explorer_cls:
outcome = await locate(ctx, state, "gear icon")
explorer_cls.assert_not_called()
assert outcome.error
assert "No screenshot" in outcome.message
@pytest.mark.asyncio
async def test_locate_contains_explorer_exceptions_as_error_outcomes():
ctx = _ctx()
state = _state()
instance = MagicMock()
instance.run = AsyncMock(side_effect=RuntimeError("quota exhausted"))
with patch("artemis.tools.explorer_tool.Explorer", return_value=instance):
outcome = await locate(ctx, state, "gear icon")
assert outcome.error and not outcome.found
assert outcome.message == "Explorer failed: quota exhausted"
assert "could not run for 'gear icon'" in render_text("gear icon", outcome, [])
# --------------------------------------------------------------------------- #
# Entry points
# --------------------------------------------------------------------------- #
@pytest.mark.asyncio
async def test_ask_explorer_text_end_to_end():
ctx = _ctx()
state = _state()
state.indexed_points = [[100, 200]]
state.indexed_elements = [{"index": 1, "center": [100, 200], "text": "Pre-existing"}]
raw = json.dumps(
{
"candidates": [
{"label": "S1", "coords": [500, 500], "description": "First Button"},
{"label": "S2", "coords": [250, 750], "description": "Second Button"},
],
"fallback_message": "Please choose carefully.",
}
)
explorer_patch, instance = _explorer_returning(raw)
with explorer_patch:
text = await ask_explorer_text(ctx, state, "Find buttons", "Attempt 1", agent_name="flash")
instance.run.assert_awaited_once()
assert instance.run.await_args.args[:2] == ("Find buttons", "Attempt 1")
assert state.indexed_points == [[100, 200], [540, 1200], [270, 1800]]
assert state.indexed_elements[1]["text"] == "First Button"
assert state.indexed_elements[2]["text"] == "Second Button"
assert "Explorer located 2 candidate(s) for 'Find buttons'" in text
assert "- [2] 'First Button' at normalized [500, 500]" in text
assert "- [3] 'Second Button' at normalized [250, 750]" in text
assert "Explorer notes: Please choose carefully." in text
@pytest.mark.asyncio
async def test_ask_explorer_text_not_found_leaves_the_state_untouched():
ctx = _ctx()
state = _state()
explorer_patch, _ = _explorer_returning(
json.dumps({"candidates": [], "fallback_message": "Covered by a keyboard."})
)
with explorer_patch:
text = await ask_explorer_text(ctx, state, "hidden button")
assert state.indexed_points == []
assert state.indexed_elements == []
assert "Explorer could not locate 'hidden button'. Covered by a keyboard." in text
@pytest.mark.asyncio
async def test_langchain_tool_uses_the_operator_presentation(tmp_path):
ctx = _ctx(base_dir=tmp_path)
state = _state(screenshot=str(tmp_path / "shot.jpg"))
raw = json.dumps({"candidates": [{"label": "S1", "coords": [500, 500], "description": "B"}]})
explorer_patch, _ = _explorer_returning(raw)
def fake_draw_dots(screenshot_path, points, labels, output_path, **kwargs):
Path(output_path).write_bytes(b"img")
with explorer_patch, patch("artemis.tools.explorer_tool.draw_dots", side_effect=fake_draw_dots):
tool = get_ask_explorer_tool(ctx)
result = await tool.ainvoke(
{"query": "Find buttons", "context_feedback": "", "state": state}
)
assert "Explorer successfully located" in result
assert "Icon" in result
assert tool.name == "ask_explorer"
assert isinstance(result, list) and result[1]["type"] == "image_url"
assert "- [1] 'B' at normalized [500, 500]" in result[0]["text"]
@pytest.mark.asyncio
async def test_direct_execute_accepts_a_version_pin_and_legacy_arg_names():
ctx = _ctx(width=None)
state = _state()
explorer_patch, instance = _explorer_returning(
json.dumps({"candidates": [], "fallback_message": "Direct execution not found."})
)
with explorer_patch:
result = await ask_explorer.execute(
ctx=ctx, state=state, Query="Find icon", ContextFeedback="cf", version="pro"
)
assert instance.run.await_args.args[:2] == ("Find icon", "cf")
assert instance.run.await_args.kwargs["version"] == "pro"
assert "Direct execution not found." in result
@pytest.mark.asyncio
async def test_run_explorer_logic_replay_shim_returns_operator_blocks(tmp_path):
ctx = _ctx(base_dir=tmp_path)
state = _state(screenshot=str(tmp_path / "shot.jpg"))
raw = json.dumps({"candidates": [{"label": "S1", "coords": [500, 500], "description": "B"}]})
explorer_patch, instance = _explorer_returning(raw)
def fake_draw_dots(screenshot_path, points, labels, output_path, **kwargs):
Path(output_path).write_bytes(b"img")
with explorer_patch, patch("artemis.tools.explorer_tool.draw_dots", side_effect=fake_draw_dots):
result = await _run_explorer_logic(ctx, state, "B", "", version="ultra")
assert instance.run.await_args.kwargs["version"] == "ultra"
assert isinstance(result, list) and len(result) == 2
assert result[0]["type"] == "text" and result[1]["type"] == "image_url"
assert state.indexed_points == [[540, 1200]]
# --------------------------------------------------------------------------- #
# Bounds and de-duplication against the existing index
# --------------------------------------------------------------------------- #
def test_from_raw_parses_bounds_and_source_and_drops_invalid_bounds():
raw = json.dumps(
{
"candidates": [
{
"label": "X1",
"coords": [500, 500],
"bounds": [400, 450, 600, 550],
"source": "xml",
},
{"label": "D1", "coords": [100, 100], "bounds": [600, 550, 400, 450]},
{"label": "D2", "coords": [100, 200], "bounds": [0, 0, 1200, 10]},
]
}
)
outcome = ExplorerOutcome.from_raw(raw)
assert outcome.candidates[0].bounds == (400, 450, 600, 550)
assert outcome.candidates[0].source == "xml"
assert outcome.candidates[1].bounds is None
assert outcome.candidates[2].bounds is None
assert outcome.candidates[1].source == ""
def test_register_candidates_reuses_an_indexed_element_that_contains_the_point():
ctx = _ctx()
state = _state(raw={"width": 1000, "height": 2000})
state.indexed_points = [[300, 400], [800, 1600]]
state.indexed_elements = [
{"index": 1, "center": [300, 400], "text": "Gmail", "bounds": [200, 300, 400, 500]},
{"index": 2, "center": [800, 1600], "text": "Send", "bounds": [700, 1500, 900, 1700]},
]
outcome = ExplorerOutcome(candidates=[ExplorerCandidate("D1", (310, 210), "Gmail app icon")])
registered = register_candidates(ctx, state, outcome)
assert registered == [
RegisteredCandidate(
index=1,
pixel=(310, 420),
coords=(310, 210),
description="Gmail app icon",
bounds=(200, 300, 400, 500),
reused=True,
)
]
assert len(state.indexed_points) == 2 # nothing appended
assert "already in your indexed list" in render_text("Gmail app icon", outcome, registered)
def test_register_candidates_reuses_a_bare_center_within_the_dedup_radius_only():
ctx = _ctx()
state = _state(raw={"width": 1000, "height": 2000})
state.indexed_points = [[500, 1000]]
state.indexed_elements = [
{"index": 1, "center": [500, 1000], "text": "Earlier candidate", "bounds": None}
]
near = ExplorerCandidate("D1", (510, 505), "Same spot") # 10 x 10 px away
far = ExplorerCandidate("D2", (560, 500), "Sixty px right")
registered = register_candidates(ctx, state, ExplorerOutcome(candidates=[near, far]))
assert [r.reused for r in registered] == [True, False]
assert registered[0].index == 1
assert registered[1].index == 2
assert state.indexed_points == [[500, 1000], [560, 1000]]
def test_register_candidates_stores_pixel_bounds_and_ocr_flag():
ctx = _ctx()
state = _state(raw={"width": 1000, "height": 2000})
outcome = ExplorerOutcome(
candidates=[
ExplorerCandidate("O3", (500, 500), "Search", bounds=(400, 450, 600, 550), source="ocr")
]
)
registered = register_candidates(ctx, state, outcome)
assert registered[0].bounds == (400, 900, 600, 1100)
assert registered[0].reused is False
element = state.indexed_elements[0]
assert element["bounds"] == [400, 900, 600, 1100]
assert element["is_ocr"] is True
tests/unit/tools/test_history_recall.py
+7 -4
View File
@@ -108,9 +108,7 @@ def test_keyword_search_finds_step_and_carries_step_number(engine):
def test_step_range_filters_matches(engine):
out = search_history(
engine, query="popup", step_range=[1, 2], recall_config=_cfg()
)
out = search_history(engine, query="popup", step_range=[1, 2], recall_config=_cfg())
# Step 3 (the popup step) is outside the range: only the range ledger and
# a no-match line may appear.
assert "A promo popup appeared" not in out
@@ -199,7 +197,11 @@ def test_images_only_for_a_single_step_and_only_real_files(engine):
# Step 2 has a real pre and post screenshot on disk — but never more than
# one step's worth of images.
assert 1 <= len(images) <= 2
labels = [b["text"] for b in out if b.get("type") == "text" and "screenshot of Step" in b.get("text", "")]
labels = [
b["text"]
for b in out
if b.get("type") == "text" and "screenshot of Step" in b.get("text", "")
]
assert all("Step 2" in label for label in labels)
for block in images:
assert block["image_url"]["url"].startswith("data:image/jpeg;base64,")
@@ -250,6 +252,7 @@ def test_availability_requires_engine_and_config():
assert recall_history.is_available(SimpleNamespace(data_engine=None)) is False
assert recall_history.is_available(SimpleNamespace(data_engine=object())) is True
def test_foreground_app_stamp_joins_search_surface(tmp_path):
"""M5: a package-name query hits the record_step foreground_app stamp."""
engine = _make_engine(tmp_path)
tests/unit/utils/test_element_hit_test.py
+1 -3
View File
@@ -109,8 +109,6 @@ def test_graceful_degradation_to_none():
def test_element_without_optional_fields_only_reports_source():
elements = [
{"text": "", "bounds": [0, 0, 100, 100], "class": None, "resource_id": None}
]
elements = [{"text": "", "bounds": [0, 0, 100, 100], "class": None, "resource_id": None}]
semantics = hit_test_semantics(elements, 50, 50)
assert semantics == {"target_label_source": "hit_test"}
tests/unit/utils/test_task_tree.py
+92 -115
View File
@@ -14,6 +14,7 @@
import hashlib
from artemis.utils.task_tree import (
format_step_action_result,
build_plan_and_history,
get_active_subgoal_hashes,
get_recent_subgoal_hashes,
@@ -406,16 +407,40 @@ def test_build_plan_and_history_interleaved_decision_loop():
assert "* [Validator Execution Result]" not in output
def test_build_plan_and_history_safety_net_and_failure_analyzer():
def test_build_plan_and_history_safety_net_and_incident():
plan = "- [ ] Subgoal 1"
steps = [
{
"step_id": "step_1",
"step_number": 1,
"relative_time": "5.0s",
"summary": "Repaired click search",
"summary": "Tried to click search",
"action_taken": [{"action": "click", "target_text": "Search"}],
"last_execution_result": {"status": "success"},
"last_execution_result": {
"status": "failed",
"burst": False,
"execution": [
{
"action": "click",
"target_text": "Search",
"attempts": [
"Pre-execution validation failed: Target button is not visible"
],
}
],
"incident": {
"kind": "safety_net",
"category": "target_disappeared",
"reason": "Target button is not visible",
"action": {"action": "click", "target_text": "Search"},
"action_description": "Tapped 'Search' at None",
"action_index": 0,
"burst_size": 1,
"step_number": 1,
"consecutive_failures": 2,
"evidence": {},
},
},
"interleaved_events": [
{"type": "thought", "content": "Let's click search."},
# Safety net failure
@@ -429,36 +454,6 @@ def test_build_plan_and_history_safety_net_and_failure_analyzer():
"Target button is not visible",
],
},
# Failure Analyzer intervention
{
"type": "failure_analyzer_thought",
"content": ("The search button is covered by keyboard, need to go BACK first."),
},
{
"type": "tool_call",
"name": "_exec_press_key",
"args": {"key": "BACK", "state": "<State>"},
"result": ["Pressed key BACK", "<Bytes>", "/path/to/shot.jpg"],
},
{
"type": "failure_analyzer_thought",
"content": "Keyboard is closed, now re-executing search tap.",
},
{
"type": "tool_call",
"name": "_exec_click",
"args": {"target": [500, 600]},
"result": ["Tapped search button successfully", "<Bytes>"],
},
{
"type": "tool_call",
"name": "report_failure_analysis",
"args": {
"status": "fixed",
"analysis": "Closed keyboard and tapped search.",
},
"result": "Success",
},
],
}
]
@@ -472,37 +467,87 @@ def test_build_plan_and_history_safety_net_and_failure_analyzer():
" Pre-Execution Safety Net)" in output
)
assert "* [Pre-Execution Safety Net]:" in output
assert "- [Safety Net Check]: (Action not executed: Target button is not visible)" in output
assert "* [Failure Analyzer Recovery Loop]:" in output
assert " - [Tool Call]: Pressed key 'BACK'" in output
assert " - [Tool Call]: Tapped element at [500, 600]" in output
assert (
"- [Safety Net Check]: (Intercepted by Pre-Execution Safety Net: Target button is"
" not visible)" in output
)
# No repair agent: the incident IS the result line, with its escalation count.
assert "Failure Analyzer" not in output
assert (
"* [Result]: Error: Intercepted by Pre-Execution Safety Net"
" (target_disappeared, consecutive failure #2) on action `Tapped 'Search' at"
" None`: Target button is not visible" in output
)
def test_build_plan_and_history_repaired_result():
def test_build_plan_and_history_fast_action_burst():
plan = "- [ ] Subgoal 1"
burst = [
{"action": "click", "coordinates": [500, 900]},
{"action": "click", "coordinates": [880, 120], "target_text": "Skip"},
{"action": "press_key", "keycode": "BACK"},
]
steps = [
{
"step_id": "step_1",
"step_number": 1,
"relative_time": "2.5s",
"summary": "Tapped button with repair",
"action_taken": [{"action": "click"}],
"summary": "Woke the control bar and tapped Skip",
"action_taken": burst,
"last_execution_result": {
"status": "failed",
"burst": True,
"execution": [
{"action": "click", "coordinates": [500, 900]},
{
"action": "click",
"attempts": ["pre-validation failed"],
"repair": ("Tapped center to wake controls and then tapped target"),
}
"coordinates": [880, 120],
"target_text": "Skip",
"attempts": ["Error: tap rejected"],
},
{
"action": "press_key",
"keycode": "BACK",
"attempts": ["Skipped (burst aborted)"],
},
],
"status": "success",
"repair_status": "fixed",
"incident": {
"kind": "exec_error",
"category": "general",
"reason": "Error: tap rejected",
"action": burst[1],
"action_description": "Tapped 'Skip' at [880, 120]",
"action_index": 1,
"burst_size": 3,
"step_number": 1,
"consecutive_failures": 1,
"evidence": {},
},
},
}
]
# Detailed view: every member with its outcome.
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
assert " * [Validator Execution Result]: Action executed." not in output
assert " * [Result]: Repaired: Tapped center to wake controls and then tapped target" in output
assert (
"* [Planned Fast-Action Burst]: 3 actions fired back to back without the"
" safety net" in output
)
assert " 1. Tapped element at [500, 900] (executed)" in output
assert " 2. Tapped 'Skip' at [880, 120] (FAILED: Error: tap rejected)" in output
assert " 3. Pressed key 'BACK' (skipped)" in output
assert (
"* [Result]: Error: Execution failed (general, consecutive failure #1) on burst"
" action 2/3 `Tapped 'Skip' at [880, 120]`: Error: tap rejected; the remaining"
" burst actions were not executed" in output
)
# Compact view: the whole burst on one line.
compact = format_step_action_result(steps[0])
assert compact.startswith(
"Fast-action burst (3 actions, unvetted): Tapped element at [500, 900] ->"
" Tapped 'Skip' at [880, 120] -> Pressed key 'BACK'"
)
assert "(Execution failed: Error: tap rejected)" in compact
def test_get_recent_subgoal_hashes_robust_milestone_exclusion(tmp_path):
@@ -576,71 +621,3 @@ def test_get_recent_subgoal_hashes_robust_milestone_exclusion(tmp_path):
# 5. Older failed subgoals from previously completed milestones (hash_failed_old)
assert hash_failed_old not in keep_hashes
def test_build_plan_and_history_for_failure_analyzer():
plan_with_subgoals = """- [/] Goal 1
- [x] Subtask 1.1
- [/] Subtask 1.2"""
steps = [
{
"step_id": "step_1",
"step_number": 1,
"relative_time": "12.3s",
"summary": "Tapped full screen",
"action_taken": [{"action": "click", "target_text": "Fullscreen"}],
"operator_raw_thinking": "Let's tap it",
"operator_native_thinking": "Thinking to tap",
"last_execution_result": {"status": "failed"},
"interleaved_events": [
{"type": "thought", "content": "I should find fullscreen icon."},
{
"type": "tool_call",
"name": "safety_net_validation",
"args": {"target": "Fullscreen"},
"result": [
False,
"TARGET_DISAPPEARED",
"Target button is not visible",
],
},
],
}
]
# Test with subgoals and for_failure_analyzer=True
output_analyzer_with_plan = build_plan_and_history(
plan_with_subgoals,
steps,
"default",
last_n_detailed=1,
for_failure_analyzer=True,
)
assert "--- Task Plan ---" in output_analyzer_with_plan
assert (
"*(Note: Provided for context only. Do not execute the remaining"
" plan.)*" in output_analyzer_with_plan
)
assert "--- Execution History ---" in output_analyzer_with_plan
assert (
"- **Step 1 (Most Recent Step (Failed to execute, this is the step you"
" need to focus on and repair), Start: 12.3s)**" in output_analyzer_with_plan
)
assert "[Operator Decision Loop]" in output_analyzer_with_plan
assert "I should find fullscreen icon." in output_analyzer_with_plan
assert (
"* [Planned Action]: Tapped 'Fullscreen' at None (Intercepted by"
" Pre-Execution Safety Net)" in output_analyzer_with_plan
)
# Test with empty plan and for_failure_analyzer=True (should completely omit plan block)
output_analyzer_empty_plan = build_plan_and_history(
"", steps, "default", last_n_detailed=1, for_failure_analyzer=True
)
assert "--- Task Plan ---" not in output_analyzer_empty_plan
assert (
"*(Note: Provided for context only. Do not execute the remaining"
" plan.)*" not in output_analyzer_empty_plan
)
assert "--- Execution History ---" in output_analyzer_empty_plan