test: add comprehensive unit, integration, and e2e test suites

Introduce complete test coverage for the ARTEMIS platform across multiple layers: Unit tests: Add tests for agent nodes (Planner, Operator, Validator, Diagnoser, FlashRunner, Explorer), tools, utils (OCR/XML fusion, UI hierarchy), and data engine. Integration tests: Add live tool verification, reactive runner, IME, and standalone workflow tests. E2E tests: Add end-to-end scenario tests for media playback and failure recovery flows. Fixtures & mocks: Add mock inputs (audio, video, screenshot) and test utilities.
2026-08-13 12:18:32 -07:00
1 parent c1803fcb86
commit 5d2283deaa
@@ -0,0 +1,289 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unit tests for VisualStepSummarizer and Context Compressor in Flash profile."""
+
+import asyncio
+from unittest.mock import AsyncMock, Mock
+import pytest
+from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
+
+from artemis.agents.flash.context_compressor import compress_flash_messages
+from artemis.agents.flash.summarizer import VisualStepSummarizer
+from artemis.context import ArtemisContext
+from artemis.sdk.builders import Builders
+
+
+@pytest.fixture
+def mock_context():
+    ctx = Mock(spec=ArtemisContext)
+    ctx.data_engine = None
+    return ctx
+
+
+@pytest.mark.asyncio
+async def test_summarizer_dispatch_and_caching(mock_context):
+    """Verify non-blocking dispatch and summary generation."""
+    summarizer = VisualStepSummarizer(mock_context)
+
+    mock_llm_response = AIMessage(
+        content="[Step 1 State: Tapped 'Login' button. Home view opened.]"
+    )
+    summarizer._llm = Mock()
+    summarizer._llm.ainvoke = AsyncMock(return_value=mock_llm_response)
+
+    summarizer.dispatch(
+        step_number=1,
+        action_name="click",
+        action_args={"target": [500, 600]},
+        pre_img_bytes=b"fake_pre",
+        post_img_bytes=b"fake_post",
+        exec_outcome="Clicked successfully",
+    )
+
+    # Await background task flush
+    await summarizer.flush()
+
+    assert summarizer.has_summary(1)
+    summary = summarizer.get_summary(1)
+    assert "[Step 1 State: Tapped 'Login' button. Home view opened.]" in summary
+
+    # Verify memory optimization: image buffers are released
+    assert summarizer._step_inputs[1]["pre_img_bytes"] is None
+    assert summarizer._step_inputs[1]["post_img_bytes"] is None
+
+
+@pytest.mark.asyncio
+async def test_summarizer_retrigger_stalled_steps(mock_context):
+    """Verify that earlier stalled steps are re-triggered when subsequent steps arrive."""
+    summarizer = VisualStepSummarizer(mock_context)
+    attempt_count = 0
+
+    async def mock_ainvoke(messages):
+        nonlocal attempt_count
+        attempt_count += 1
+        # Fail the first attempt (Step 1 first call)
+        if attempt_count == 1:
+            raise asyncio.CancelledError("Simulated timeout")
+        # Check messages for step number
+        msg_str = str(messages)
+        if "Step 1" in msg_str:
+            return AIMessage(content="[Step 1 State: Recovered Step 1 summary]")
+        return AIMessage(content="[Step 2 State: Step 2 summary]")
+
+    summarizer._llm = Mock()
+    summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
+
+    # Dispatch step 1 (fails on first attempt)
+    summarizer.dispatch(
+        step_number=1,
+        action_name="click",
+        action_args={"target": [100, 100]},
+        pre_img_bytes=b"pre1",
+        post_img_bytes=b"post1",
+        exec_outcome="Outcome 1",
+    )
+    await summarizer.flush()
+    assert not summarizer.has_summary(1)
+
+    # Dispatch step 2 (should detect stalled Step 1 and re-trigger)
+    summarizer.dispatch(
+        step_number=2,
+        action_name="swipe",
+        action_args={"action": "up"},
+        pre_img_bytes=b"pre2",
+        post_img_bytes=b"post2",
+        exec_outcome="Outcome 2",
+    )
+    await summarizer.flush()
+
+    assert summarizer.has_summary(1)
+    assert summarizer.has_summary(2)
+    assert "Recovered Step 1" in summarizer.get_summary(1)
+
+
+def test_context_compressor_with_ready_summaries(mock_context):
+    """Verify that ready summaries replace past image blocks and prune outdated XML."""
+    summarizer = VisualStepSummarizer(mock_context)
+    summarizer._summaries[1] = "[Step 1 State: Tapped Search bar. Keyboard appeared.]"
+    summarizer._summaries[2] = "[Step 2 State: Typed 'flights'. Suggestions displayed.]"
+
+    messages = [
+        HumanMessage(
+            content=[
+                {"type": "text", "text": "Task: Search flights"},
+                {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}},
+                {"type": "text", "text": "--- UI Element List ---\nOld Tree 1"},
+            ]
+        ),
+        ToolMessage(
+            tool_call_id="tc1",
+            name="click",
+            content=[
+                {"type": "text", "text": "Action 'click' completed."},
+                {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_2"}},
+                {"type": "text", "text": "--- UI Element List ---\nOld Tree 2"},
+            ],
+        ),
+        ToolMessage(
+            tool_call_id="tc2",
+            name="input_text",
+            content=[
+                {"type": "text", "text": "Action 'input_text' completed."},
+                {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_3"}},
+                {"type": "text", "text": "--- UI Element List ---\nLive Latest Tree 3"},
+            ],
+        ),
+    ]
+
+    compress_flash_messages(messages, summarizer=summarizer, prune_history_xml=True)
+
+    # Initial HumanMessage: Historical initial image pruned, task text preserved
+    assert len(messages[0].content) == 1
+    assert messages[0].content[0]["text"] == "Task: Search flights"
+
+    # Step 1 (ToolMessage 1): Image replaced with Step 1 summary, old XML pruned
+    assert len(messages[1].content) == 2
+    assert messages[1].content[0]["text"] == "Action 'click' completed."
+    assert messages[1].content[1]["text"] == "[Step 1 State: Tapped Search bar. Keyboard appeared.]"
+
+    # Step 2 (ToolMessage 2 - Latest live screen): Image and latest live XML MUST be preserved
+    assert len(messages[2].content) == 3
+    assert messages[2].content[0]["text"] == "Action 'input_text' completed."
+    assert messages[2].content[1]["type"] == "image_url"
+    assert "Live Latest Tree 3" in messages[2].content[2]["text"]
+
+
+def test_context_compressor_graceful_fallback(mock_context):
+    """Verify that when a summary is not ready yet, it cleanly falls back to standard pruned state."""
+    summarizer = VisualStepSummarizer(mock_context)
+    # Step 1 has no summary in cache
+
+    messages = [
+        HumanMessage(
+            content=[
+                {"type": "text", "text": "Task: Search"},
+                {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}},
+            ]
+        ),
+        ToolMessage(
+            tool_call_id="tc1",
+            name="click",
+            content=[
+                {"type": "text", "text": "Action 'click' completed."},
+                {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_2"}},
+            ],
+        ),
+    ]
+
+    compress_flash_messages(messages, summarizer=summarizer, prune_history_xml=True)
+
+    # Turn 1 should simply have image omitted (standard fallback)
+    assert len(messages[0].content) == 1
+    assert messages[0].content[0]["text"] == "Task: Search"
+
+    # Turn 2 (Latest) preserves image
+    assert len(messages[1].content) == 2
+    assert messages[1].content[1]["type"] == "image_url"
+
+
+def test_flash_config_and_builder():
+    """Verify Flash profile configuration model and SDK builder fluent API."""
+    cfg = Builders.AgentConfig.with_flash_config(
+        max_turns=25,
+        explorer_mode="flash",
+        step_summarizer=True,
+        step_summarizer_model="gemini-2.5-flash-lite",
+        prune_history_xml=True,
+    ).build()
+
+    assert cfg.flash.max_turns == 25
+    assert cfg.flash.explorer_mode == "flash"
+    assert cfg.flash.step_summarizer.enabled is True
+    assert cfg.flash.step_summarizer.model == "gemini-2.5-flash-lite"
+    assert cfg.flash.step_summarizer.prune_history_xml is True
+
+
+@pytest.mark.asyncio
+async def test_data_engine_integration_with_step_summarizer(mock_context):
+    """Verify that generated summaries sync into DataEngine and are visible to get_agent_friendly_steps."""
+    mock_de = Mock()
+    mock_de.update_step_summary = Mock()
+    mock_context.data_engine = mock_de
+
+    summarizer = VisualStepSummarizer(mock_context)
+    summarizer._llm = Mock()
+    summarizer._llm.ainvoke = AsyncMock(
+        return_value=AIMessage(content="[Step 1 State: Input text confirmed.]")
+    )
+
+    summarizer.dispatch(
+        step_number=1,
+        action_name="input_text",
+        action_args={"text": "hello"},
+        pre_img_bytes=b"pre",
+        post_img_bytes=b"post",
+        exec_outcome="Executed typing",
+    )
+    await summarizer.flush()
+
+    mock_de.update_step_summary.assert_called_once_with(1, "[Step 1 State: Input text confirmed.]")
+
+
+def test_draw_action_overlay_rendering():
+    """Verify that draw_action_overlay_on_image correctly renders markers on images."""
+    import io
+    from PIL import Image
+    from artemis.utils.visualization import draw_action_overlay_on_image
+
+    # Create a blank RGB image in memory
+    img = Image.new("RGB", (1080, 2400), color="white")
+    buf = io.BytesIO()
+    img.save(buf, format="JPEG")
+    raw_bytes = buf.getvalue()
+
+    # 1. Click overlay
+    annotated_click = draw_action_overlay_on_image(
+        image_bytes=raw_bytes,
+        action_name="click",
+        action_args={"target": [540, 1200]},
+    )
+    assert len(annotated_click) > 0
+    assert annotated_click != raw_bytes
+
+    # 2. Swipe overlay
+    annotated_swipe = draw_action_overlay_on_image(
+        image_bytes=raw_bytes,
+        action_name="swipe",
+        action_args={"start": [500, 1800], "end": [500, 600]},
+    )
+    assert len(annotated_swipe) > 0
+    assert annotated_swipe != raw_bytes
+
+    # 3. Click Sequence overlay
+    annotated_seq = draw_action_overlay_on_image(
+        image_bytes=raw_bytes,
+        action_name="click_sequence",
+        action_args={"sequence": [[200, 300], [400, 500], [600, 700]]},
+    )
+    assert len(annotated_seq) > 0
+    assert annotated_seq != raw_bytes
+
+    # 4. Fail-safe on corrupt bytes
+    corrupt_result = draw_action_overlay_on_image(
+        image_bytes=b"invalid_bytes",
+        action_name="click",
+        action_args={"target": [100, 100]},
+    )
+    assert corrupt_result == b"invalid_bytes"