test: add comprehensive unit, integration, and e2e test suites
Introduce complete test coverage for the ARTEMIS platform across multiple layers: Unit tests: Add tests for agent nodes (Planner, Operator, Validator, Diagnoser, FlashRunner, Explorer), tools, utils (OCR/XML fusion, UI hierarchy), and data engine. Integration tests: Add live tool verification, reactive runner, IME, and standalone workflow tests. E2E tests: Add end-to-end scenario tests for media playback and failure recovery flows. Fixtures & mocks: Add mock inputs (audio, video, screenshot) and test utilities.
This commit is contained in:
chiwanko11 committed
1 parent
c1803fcb86
commit
5d2283deaa
115 files changed
+22813
No files matched your search
@@ -0,0 +1,213 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for Universal Failure Analyzer."""
|
||||
|
||||
import base64
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage
|
||||
|
||||
from artemis.agents.validator.failure_analyzer import (
|
||||
FailureAnalyzer,
|
||||
PixelTargetDisappearedStrategy,
|
||||
TargetDisappearedStrategy,
|
||||
ValidationErrorCategory,
|
||||
VALIDATOR_TOOLS_DECLARATION,
|
||||
)
|
||||
from artemis.context import ArtemisContext
|
||||
|
||||
|
||||
class DummyState:
|
||||
def __init__(self, latest_screenshot=None):
|
||||
self.latest_screenshot = latest_screenshot
|
||||
self.indexed_points = []
|
||||
self.indexed_elements = []
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context(tmp_path):
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.llm_config = Mock()
|
||||
mock_llm_cfg = Mock()
|
||||
mock_llm_cfg.model = "gemini-2.5-pro"
|
||||
mock_llm_cfg.temperature = 0.1
|
||||
ctx.llm_config.get_agent.return_value = mock_llm_cfg
|
||||
|
||||
ctx.device = Mock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
|
||||
ctx.data_engine = Mock()
|
||||
ctx.data_engine.base_dir = tmp_path
|
||||
ctx.data_engine.get_agent_friendly_steps.return_value = []
|
||||
ctx.adb_client = None
|
||||
ctx.driver = Mock()
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_analyzer_cannot_fix(mock_context, tmp_path):
|
||||
"""Test that FailureAnalyzer natively executes and handles 'cannot_fix' report."""
|
||||
screenshot_file = tmp_path / "shot.jpg"
|
||||
screenshot_file.write_bytes(b"dummy_bytes")
|
||||
|
||||
state = DummyState(latest_screenshot=str(screenshot_file))
|
||||
failed_action = {"action": "click", "coordinates": [100, 200]}
|
||||
error_msg = "Element not found"
|
||||
|
||||
analyzer = FailureAnalyzer(mock_context)
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_response = AIMessage(
|
||||
content="I see the error.",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "report_failure_analysis",
|
||||
"args": {
|
||||
"status": "cannot_fix",
|
||||
"analysis": "The element was definitely not there.",
|
||||
},
|
||||
"id": "call_1",
|
||||
}
|
||||
],
|
||||
)
|
||||
mock_bound = MagicMock()
|
||||
mock_bound.ainvoke = AsyncMock(return_value=mock_response)
|
||||
mock_llm.bind_tools.return_value = mock_bound
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.get_llm", return_value=mock_llm),
|
||||
patch(
|
||||
"artemis.agents.validator.failure_analyzer.UnifiedMobileController"
|
||||
) as mock_controller_cls,
|
||||
):
|
||||
mock_controller = Mock()
|
||||
mock_controller_cls.return_value = mock_controller
|
||||
mock_device_data = Mock()
|
||||
mock_device_data.base64 = base64.b64encode(b"mocked_screenshot").decode()
|
||||
mock_device_data.elements = []
|
||||
mock_device_data.width = 1080
|
||||
mock_device_data.height = 2400
|
||||
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
|
||||
mock_controller.get_ui_elements = AsyncMock(return_value=mock_device_data.elements)
|
||||
|
||||
result = await analyzer.analyze(
|
||||
state,
|
||||
failed_action,
|
||||
error_msg,
|
||||
pre_screenshot=base64.b64encode(b"dummy").decode(),
|
||||
post_screenshot=base64.b64encode(b"dummy").decode(),
|
||||
)
|
||||
|
||||
assert result["status"] == "cannot_fix"
|
||||
assert "not there" in result["analysis"]
|
||||
assert result["new_remaining_actions"] == []
|
||||
|
||||
|
||||
def test_report_failure_analysis_tool_declaration():
|
||||
"""Test that report_failure_analysis tool declaration is correct and matches."""
|
||||
report_tool = next(
|
||||
(t for t in VALIDATOR_TOOLS_DECLARATION if t.name == "report_failure_analysis"),
|
||||
None,
|
||||
)
|
||||
assert report_tool is not None
|
||||
props = report_tool.parameters.get("properties", {})
|
||||
assert "analysis" in props
|
||||
assert "single paragraph" in props["analysis"]["description"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_analyzer_strategy_routing(mock_context):
|
||||
"""Test that FailureAnalyzer selects appropriate strategies based on error category."""
|
||||
analyzer = FailureAnalyzer(mock_context)
|
||||
|
||||
strat_disappeared = analyzer._select_strategy(ValidationErrorCategory.TARGET_DISAPPEARED)
|
||||
assert isinstance(strat_disappeared, TargetDisappearedStrategy)
|
||||
assert "target_disappeared" in strat_disappeared.get_prompt_template_name()
|
||||
|
||||
strat_pixel = analyzer._select_strategy(ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED)
|
||||
assert isinstance(strat_pixel, PixelTargetDisappearedStrategy)
|
||||
assert "pixel_target_disappeared" in strat_pixel.get_prompt_template_name()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_analyzer_click_sequence(mock_context, tmp_path):
|
||||
"""Test click_sequence execution in FailureAnalyzer."""
|
||||
screenshot_file = tmp_path / "shot.jpg"
|
||||
screenshot_file.write_bytes(b"dummy_bytes")
|
||||
|
||||
state = DummyState(latest_screenshot=str(screenshot_file))
|
||||
analyzer = FailureAnalyzer(mock_context)
|
||||
|
||||
# Turn 1: Call click_sequence tool with direct coordinates
|
||||
call_step_1 = AIMessage(
|
||||
content="Let me click in sequence.",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "click_sequence",
|
||||
"args": {"sequence": [[100, 200], [300, 400]]},
|
||||
"id": "call_seq_1",
|
||||
}
|
||||
],
|
||||
)
|
||||
# Turn 2: Report failure fixed
|
||||
call_step_2 = AIMessage(
|
||||
content="Repaired successfully.",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "report_failure_analysis",
|
||||
"args": {
|
||||
"status": "fixed",
|
||||
"analysis": "Sequence clicked successfully.",
|
||||
},
|
||||
"id": "call_report_2",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_bound = MagicMock()
|
||||
mock_bound.ainvoke = AsyncMock(side_effect=[call_step_1, call_step_2])
|
||||
mock_llm.bind_tools.return_value = mock_bound
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.get_llm", return_value=mock_llm),
|
||||
patch(
|
||||
"artemis.agents.validator.failure_analyzer.UnifiedMobileController"
|
||||
) as mock_controller_cls,
|
||||
):
|
||||
mock_controller = Mock()
|
||||
mock_controller_cls.return_value = mock_controller
|
||||
mock_device_data = Mock()
|
||||
mock_device_data.base64 = base64.b64encode(b"mocked_screenshot").decode()
|
||||
mock_device_data.elements = []
|
||||
mock_device_data.width = 1080
|
||||
mock_device_data.height = 2400
|
||||
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
|
||||
mock_controller.get_ui_elements = AsyncMock(return_value=mock_device_data.elements)
|
||||
mock_controller.tap_at = AsyncMock(return_value=Mock(error=None))
|
||||
|
||||
result = await analyzer.analyze(
|
||||
state,
|
||||
{"action": "click", "coordinates": [100, 200]},
|
||||
"Element not found",
|
||||
pre_screenshot=base64.b64encode(b"dummy").decode(),
|
||||
post_screenshot=base64.b64encode(b"dummy").decode(),
|
||||
error_category=ValidationErrorCategory.TARGET_DISAPPEARED,
|
||||
)
|
||||
|
||||
assert result["status"] == "fixed"
|
||||
assert mock_controller.tap_at.call_count == 2
|
||||
Reference in new issue
Block a user