test: add comprehensive unit, integration, and e2e test suites

Introduce complete test coverage for the ARTEMIS platform across multiple layers:
Unit tests: Add tests for agent nodes (Planner, Operator, Validator, Diagnoser, FlashRunner, Explorer), tools, utils (OCR/XML fusion, UI hierarchy), and data engine.
Integration tests: Add live tool verification, reactive runner, IME, and standalone workflow tests.
E2E tests: Add end-to-end scenario tests for media playback and failure recovery flows.
Fixtures & mocks: Add mock inputs (audio, video, screenshot) and test utilities.
This commit is contained in:
chiwanko11 committed 2026-08-13 12:18:32 -07:00
1 parent c1803fcb86
commit 5d2283deaa
115 files changed
+22813

No files matched your search

@@ -0,0 +1,124 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import logging
import os
import sys
from unittest.mock import patch
from dotenv import load_dotenv
# Ensure we can import from project
sys.path.append(os.path.dirname(__file__))
from artemis.sdk import Agent
from artemis.sdk.builders import Builders
from artemis.sdk.types import AgentProfile
from artemis.agents.validator.validator import ValidatorNode, ValidationErrorCategory
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("test_fa_e2e")
# We want to fail the first validation to trigger the Failure Analyzer
original_validate = ValidatorNode._validate_action_precondition
original_validate_pixel = ValidatorNode._validate_action_precondition_pixel
has_failed_once = False
async def mock_validate_action_precondition(self, session, action_item, state=None):
global has_failed_once
if not has_failed_once:
has_failed_once = True
logger.warning(">>> MOCK VALIDATOR: Simulating pre-execution XML validation failure!")
return (
False,
ValidationErrorCategory.TARGET_DISAPPEARED,
"Simulated pre-execution XML validation failure",
)
return await original_validate(self, session, action_item, state=state)
async def mock_validate_action_precondition_pixel(
self,
session,
action_item,
pre_screenshot_b64,
original_coords=None,
state=None,
):
global has_failed_once
if not has_failed_once:
has_failed_once = True
logger.warning(">>> MOCK VALIDATOR: Simulating pre-execution Pixel validation failure!")
return (
False,
ValidationErrorCategory.TARGET_DISAPPEARED,
"Simulated pre-execution Pixel validation failure",
)
return await original_validate_pixel(
self,
session,
action_item,
pre_screenshot_b64,
original_coords,
state=state,
)
async def main():
load_dotenv()
# Check if emulator/device is connected
import subprocess
res = subprocess.run(["adb", "devices"], capture_output=True, text=True)
print("ADB Devices:")
print(res.stdout)
# 1. Initialize agent
logger.info("Initializing Agent...")
profile = AgentProfile(name="default", from_file="llm-config.override.jsonc")
config = Builders.AgentConfig.with_default_profile(profile).build()
agent = Agent(config=config)
await agent.init()
# 2. Run the task with patches
goal = "Play a video in full screen on YouTube in Chrome"
logger.info(f"Running task with goal: {goal}")
with (
patch.object(
ValidatorNode,
"_validate_action_precondition",
mock_validate_action_precondition,
),
patch.object(
ValidatorNode,
"_validate_action_precondition_pixel",
mock_validate_action_precondition_pixel,
),
):
try:
result = await agent.run_task(goal=goal, name="test_fa_youtube_fullscreen")
logger.info(f"Task finished! Result: {result}")
except Exception:
logger.exception("Error during execution:")
finally:
await agent.clean()
if __name__ == "__main__":
asyncio.run(main())