feat(android): add native controls and continuous multi-touch

tests/unit/agents/test_checker_native_evidence.py

@@ -0,0 +1,103 @@
+"""Guard the prompt evidence contract; no model or device is invoked here."""
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from artemis.agents.checker.checker import (
+    CheckReport,
+    assemble_checker_prompt_segments,
+    run_final_check,
+)
+from artemis.context import ArtemisContext, ExecutionSetup
+
+
+@pytest.mark.parametrize("entry", ["checkpoint", "final"])
+def test_bound_persistent_readout_can_supply_precision_beyond_pixel_ticks(entry):
+    prompt = assemble_checker_prompt_segments(entry, [], probe_tool_registered=False)
+
+    # Native origin alone is insufficient: the rule must bind the displayed value
+    # to the selected state and require stability plus the visible pointer check.
+    for requirement in (
+        "标签、节点角色/层级",
+        "目标控件当前所选值或固定指针所选时间",
+        "不能仅因原生节点含有数值或执行者称其可信",
+        "在被验收状态中持续显示",
+        "引用已停稳或暂停的证据",
+        "与同一控件的指针及可辨刻度一致",
+        "结合读数精度与用户容差",
+        "粗刻度无法独立区分秒数，不等于",
+        "无需再由粗像素刻度独立插值证明同一精确值",
+    ):
+        assert requirement in prompt
+
+    assert "拖动浮层或原生时间读数与固定指针刻度冲突时" not in prompt
+
+
+@pytest.mark.parametrize("entry", ["checkpoint", "final"])
+def test_native_origin_does_not_promote_unbound_or_conflicting_evidence(entry):
+    prompt = assemble_checker_prompt_segments(entry, [], probe_tool_registered=False)
+
+    for excluded_source in ("事件卡片时间", "视频画面水印", "录制起止时间", "拖动/松手浮层"):
+        assert excluded_source in prompt
+    assert "即使它们也能通过原生节点读取，仍须先证明上述绑定与持久性" in prompt
+    assert "可辨精度上存在实际冲突" in prompt
+    assert "读数仍在变化、绑定不明或无法确认已停稳/暂停" in prompt
+    assert "应判证据不足（inconclusive）" in prompt
+    assert "不得挑选有利读数或忽略矛盾" in prompt
+    assert "可信读数明确超出用户容差时仍按通用规则判定" in prompt
+
+
+@pytest.mark.parametrize("entry", ["checkpoint", "final"])
+def test_pixel_fallback_remains_quantitative_and_requires_post_action_evidence(entry):
+    prompt = assemble_checker_prompt_segments(entry, [], probe_tool_registered=False)
+    fallback = prompt.split("没有可信持久读数时，", 1)[1]
+
+    assert "动作后已停稳截图的两处真实数值刻度与固定指针位置作线性插值" in fallback
+    assert "单位换算、估读不确定性及目标误差" in fallback
+    assert "目标误差加估读不确定性不超过用户容差" in fallback
+    assert "刻度非线性或无法确定已停稳，仍应判证据不足" in fallback
+    assert "操作前的校准计划及回执误差不能替代动作后测量" in fallback
+
+
+@pytest.mark.asyncio
+async def test_final_review_receives_live_readout_and_rules_separately_from_report_claims():
+    ctx = MagicMock(spec=ArtemisContext)
+    ctx.execution_setup = ExecutionSetup()
+    ctx.data_engine = MagicMock()
+    ctx.data_engine.base_dir = "unused"
+    ctx.data_engine.get_agent_friendly_steps.return_value = []
+    ctx.data_engine.get_step_number.return_value = 7
+    elements = "[1] 当前所选值: 23.5; [2] 已暂停"
+    review = AsyncMock(return_value=CheckReport(verdicts=[]))
+
+    with (
+        patch("artemis.agents.checker.checker._run_check_loop", new=review),
+        patch(
+            "artemis.agents.checker.checker._capture_final_screen",
+            new=AsyncMock(return_value=("captured-image", elements)),
+        ),
+    ):
+        await run_final_check(
+            ctx,
+            goal="核对当前所选值及容差",
+            plan_text="",
+            ledger=[],
+            check_items=[],
+            execution_report="执行者声称当前所选值为 88",
+        )
+
+    review.assert_awaited_once()
+    messages = review.await_args.args[1]
+    assert "无需再由粗像素刻度独立插值证明同一精确值" in messages[0].content
+    assert "不得挑选有利读数或忽略矛盾" in messages[0].content
+    content = messages[1].content
+    text = content[0]["text"]
+    assert "Executor's Final Report (claims to verify, not independent evidence)" in text
+    assert "执行者声称当前所选值为 88" in text
+    assert f"# Final Screen Elements\n{elements}" in text
+    assert any(
+        block.get("type") == "image_url"
+        and block["image_url"]["url"] == "data:image/jpeg;base64,captured-image"
+        for block in content
+    )

tests/unit/agents/test_native_receipts.py

@@ -0,0 +1,194 @@
+"""Actual action-result/history/transcript rendering must keep native evidence."""
+
+from copy import deepcopy
+import json
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
+import pytest
+
+from artemis.agents.validator.action_execution import exec_action
+from artemis.mcp.action_types import ActionCode, ActionResult
+from artemis.memory.transcript import EXECUTION_RESULT_MARKER, TranscriptLedger
+from artemis.utils.task_tree import format_result_clean
+
+
+NATIVE_ACTIONS = (
+    "pinch",
+    "touch_gesture",
+    "read_control",
+    "control_action",
+    "wait_until",
+    "scroll_to",
+)
+TOKEN = "a" * 64
+SELECTOR = {"resource_id": "fixture:id/editor"}
+
+
+def action_arguments(action):
+    drag = {
+        "start": [191, 501],
+        "end": [192, 532],
+        "coordinate_space": "pixel",
+        "target_description": "observed canvas",
+    }
+    return {
+        "pinch": {
+            "center": [200, 400],
+            "start_span": 100,
+            "end_span": 200,
+            "coordinate_space": "pixel",
+            "target_description": "observed canvas",
+        },
+        "touch_gesture": {
+            "frames": [
+                {"at_ms": 0, "points": [[191, 501]]},
+                {"at_ms": 400, "points": [[192, 532]]},
+            ],
+            "coordinate_space": "pixel",
+            "target_description": "observed canvas",
+        },
+        "read_control": {"selector": SELECTOR},
+        "control_action": {
+            "selector": SELECTOR,
+            "action": "set_selection",
+            "selection_start": 3,
+            "selection_end": 5,
+        },
+        "wait_until": {"selector": SELECTOR, "state": "value", "expected_value": 37.5},
+        "scroll_to": {"container": {"resource_id": "fixture:id/list"}, "target": SELECTOR},
+        "drag": drag,
+        "action_sequence": {"steps": [{"action": "drag", **drag}]},
+    }[action]
+
+
+def control_observation():
+    return {
+        "resource_id": "fixture:id/editor",
+        "text": "  A😀B \n",
+        "selection_start": 3,
+        "selection_end": 5,
+        "range": {"min": 0, "max": 100, "current": 37.5, "type": 1},
+        "token": TOKEN,
+        "content_signature": "b" * 64,
+        "editable": True,
+        "password": False,
+    }
+
+
+def decode_receipt(text, label):
+    marker = f"{label} receipt: "
+    assert marker in text
+    return json.loads(text.split(marker, 1)[1])
+
+
+async def render_result_pipeline(action, metadata, failed):
+    message = (
+        "Native reply is uncertain; observe before continuing." if failed else "Input accepted."
+    )
+    result = (
+        ActionResult.failure(action, message, code=ActionCode.TIMEOUT, metadata=metadata)
+        if failed
+        else ActionResult.success(action, message, metadata=metadata)
+    )
+    session = SimpleNamespace(call=AsyncMock(return_value=result))
+    context = SimpleNamespace(
+        data_engine=None, device=SimpleNamespace(device_width=400, device_height=800)
+    )
+    args = action_arguments(action)
+    item = {"action": action, "args": args}
+    ok, error = await exec_action(context, session, item)
+    assert ok is not failed
+    session.call.assert_awaited_once_with(action, args)
+    assert item["result"] == result.agent_message
+    report = {
+        "status": "failed" if failed else "dispatched",
+        "execution": [{**item, "attempts": [f"Error: {error}" if failed else "Dispatched"]}],
+    }
+    history = format_result_clean(report)
+    assert isinstance(history, str)
+
+    ledger = TranscriptLedger(clock=lambda: 0.0)
+    ledger.stage_turn(
+        [
+            HumanMessage(content="Observed native controls"),
+            AIMessage(content="", tool_calls=[{"name": action, "args": {}, "id": "native-call"}]),
+            ToolMessage(content="Action Recorded", tool_call_id="native-call"),
+        ]
+    )
+    ledger.commit_staged(step_key="native-step", validator_result=report)
+    rendered = ledger.render([HumanMessage(content="Next observation")])
+    execution_results = [
+        message.content[0]["text"]
+        for message in rendered
+        if isinstance(message, HumanMessage)
+        and isinstance(message.content, list)
+        and message.content
+        and EXECUTION_RESULT_MARKER in message.content[0].get("text", "")
+    ]
+    assert len(execution_results) == 1
+    transcript = execution_results[0]
+    assert f"Status: {report['status']}" in transcript
+    if failed:
+        assert "Error:" in history and "Error:" in transcript
+    return result.agent_message, history, transcript
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("action", NATIVE_ACTIONS)
+@pytest.mark.parametrize("failed", [False, True])
+async def test_every_native_action_keeps_readings_selection_range_and_token_in_actual_result_rendering(
+    action, failed
+):
+    node = control_observation()
+    read_only = action in {"read_control", "wait_until"}
+    execution = (
+        ("not_started" if read_only else "unknown")
+        if failed
+        else ("observed" if read_only else "completed")
+    )
+    # The renderer must retain optional observed evidence even on gesture receipts.
+    metadata = {
+        "action": action,
+        "execution": execution,
+        "node": deepcopy(node),
+        "native_receipt": {
+            "success": not failed,
+            "execution": execution,
+            "code": "TIMEOUT" if failed else "OK",
+            "node": deepcopy(node),
+        },
+        "verification": "pending_observation",
+    }
+    stages = await render_result_pipeline(action, metadata, failed)
+    for rendered in stages:
+        recovered = decode_receipt(rendered, "Native")
+        assert recovered == metadata
+        assert recovered["node"]["text"] == "  A😀B \n"
+        assert recovered["node"]["range"]["current"] == 37.5
+        assert (recovered["node"]["selection_start"], recovered["node"]["selection_end"]) == (3, 5)
+        assert recovered["node"]["token"] == TOKEN
+        assert recovered["native_receipt"]["success"] is not failed
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("action", ["drag", "action_sequence"])
+@pytest.mark.parametrize("failed", [False, True])
+async def test_existing_drag_and_sequence_geometry_and_partial_receipts_still_render(
+    action, failed
+):
+    metadata = {
+        "execution": "partial" if failed else "completed",
+        "coordinate_space": "pixel",
+        "start": [191, 501],
+        "end": [192, 532],
+        "calibration": {"axis": "y", "target_value": 37.5},
+    }
+    if action == "action_sequence":
+        metadata["sequence_results"] = [
+            {"action": "click", "execution": "completed", "completed_taps": 1},
+            {"action": "drag", "execution": "unknown" if failed else "completed"},
+        ]
+    for rendered in await render_result_pipeline(action, metadata, failed):
+        assert decode_receipt(rendered, "Gesture") == metadata

tests/unit/agents/test_operator_transcript.py

@@ -35,6 +35,7 @@ from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, Tool
 from artemis.agents.operator.operator import OperatorNode
 from artemis.agents.operator.prompts import load_operator_prompts
 from artemis.agents.operator.prompts import (
+    OPERATOR_PROMPT_TOOLSET,
     PLAN_HISTORY_STATIC_POINTER,
     PLAN_HISTORY_TEMPLATE_SECTION,
     PromptBuilder,
@@ -54,8 +55,8 @@ from artemis.memory.transcript import (
 
 # SHA-256 snapshots of the legacy system message with the fixed inputs below.
 # Update these when an intentional template change alters the rendered prompt.
-GOLDEN_EMPTY_PLAN = "23c3d6b5fc3694275e7015f7bd0901bde54aae54fec7a8c68d4f8529c65ef4bc"
-GOLDEN_SENTINEL_PLAN = "370c53f1d088a4120c63fbaea6d895f121c683b8d3c8cc7d61105d6e1d223128"
+GOLDEN_EMPTY_PLAN = "f5ee50f4923fed4cb943cb80087d39ffffe66f259cfae4a54f27a244d7b06453"
+GOLDEN_SENTINEL_PLAN = "bb725e2d096f6aa0cf1773809b2d0e672475589782b9c72348d9eaf8481230b9"
 
 SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
 
@@ -84,6 +85,42 @@ async def test_legacy_prompt_bytes_match_chinese_prompt_golden():
     assert hashlib.sha256(sentinel.encode("utf-8")).hexdigest() == GOLDEN_SENTINEL_PLAN
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("transcript", [False, True])
+async def test_legacy_and_transcript_prompts_omit_all_unavailable_native_capabilities(transcript):
+    native_tools = {
+        "pinch",
+        "touch_gesture",
+        "read_control",
+        "control_action",
+        "wait_until",
+        "scroll_to",
+    }
+    available_tools = OPERATOR_PROMPT_TOOLSET - native_tools
+    ctx = SimpleNamespace(
+        execution_setup=None,
+        actuator=SimpleNamespace(capabilities=lambda: available_tools),
+        data_engine=None,
+    )
+    state = SimpleNamespace(initial_goal="FILTERED_FIXED_GOAL")
+    if transcript:
+        system = render_transcript_static_system(load_operator_prompts(), ctx, state)
+    else:
+        builder = PromptBuilder()
+        await TemplatePromptComponent()(
+            builder,
+            state,
+            ctx,
+            prompts=load_operator_prompts(),
+            template_name="main_template",
+        )
+        system = "".join(builder.system_parts)
+    assert "FILTERED_FIXED_GOAL" in system
+    assert "`click`" in system and "`drag`" in system
+    for tool in native_tools:
+        assert tool not in system
+
+
 def test_main_template_contains_exactly_one_plan_history_section():
     template = load_operator_prompts()["main_template"]
     assert template.count(PLAN_HISTORY_TEMPLATE_SECTION) == 1

tests/unit/agents/test_prompt_assembly.py

@@ -38,6 +38,14 @@ OPERATOR_DEVICE_TOOLS = (
     "wait_for_delay",
     "long_press",
 )
+OPERATOR_NATIVE_TOOLS = (
+    "pinch",
+    "touch_gesture",
+    "read_control",
+    "control_action",
+    "wait_until",
+    "scroll_to",
+)
 
 
 def _tool_refs(text: str, tool: str) -> list[str]:
@@ -88,6 +96,7 @@ def test_full_toolset_is_the_default():
 @pytest.mark.parametrize(
     "tool",
     OPERATOR_DEVICE_TOOLS
+    + OPERATOR_NATIVE_TOOLS
     + ("video_analyzer", "search_history", "replay_steps", "get_step_screenshot"),
 )
 def test_removed_tool_leaves_no_reference(tool):
@@ -99,6 +108,23 @@ def test_removed_tool_leaves_no_reference(tool):
         assert not _tool_refs(out, tool), f"'{tool}' still referenced in {name}"
 
 
+def test_removing_all_native_capabilities_leaves_no_native_names_in_any_template():
+    available_tools = OPERATOR_PROMPT_TOOLSET - set(OPERATOR_NATIVE_TOOLS)
+    for name, template in load_operator_prompts().items():
+        out = apply_operator_prompt_contract(template, available_tools=available_tools)
+        for tool in OPERATOR_NATIVE_TOOLS:
+            assert tool not in out, f"'{tool}' still advertised without native capability in {name}"
+        assert "`click`" in out and "`drag`" in out
+        assert (
+            "Physical device actions (`click`, `action_sequence`, `drag`, `input_text`, `swipe`,"
+            " `press_key`, `manage_app`, `wait_for_delay`, `long_press`)" in out
+        )
+        assert (
+            "Turn-Ending Action (`click`, `action_sequence`, `drag`, `swipe`, `input_text`,"
+            " `long_press`, `press_key`, `manage_app`, or `wait_for_delay`)" in out
+        )
+
+
 def test_removed_swipe_takes_its_teaching_along():
     """Gating removes the drag/slider teaching, not just the backticked name."""
     template = load_operator_prompts()["main_template"]
@@ -131,26 +157,30 @@ def test_reduced_enums_stay_well_formed():
         available_tools=OPERATOR_PROMPT_TOOLSET - {"manage_app", "wait_for_delay"},
     )
     assert (
-        "Physical device actions (`click`, `action_sequence`, `drag`, `input_text`, `swipe`, `press_key`, `long_press`)"
-        in out
+        "Physical device actions (`click`, `action_sequence`, `drag`, `pinch`, `touch_gesture`,"
+        " `read_control`, `control_action`, `wait_until`, `scroll_to`, `input_text`, `swipe`,"
+        " `press_key`, `long_press`)" in out
     )
     assert (
-        "Turn-Ending Action (`click`, `action_sequence`, `drag`, `swipe`, `input_text`, `long_press`, or `press_key`)"
-        in out
+        "Turn-Ending Action (`click`, `action_sequence`, `drag`, `pinch`, `touch_gesture`,"
+        " `read_control`, `control_action`, `wait_until`, `scroll_to`, `swipe`, `input_text`,"
+        " `long_press`, or `press_key`)" in out
     )
 
 
 def test_full_set_enum_slots_render_verbatim():
-    """With every tool present the historical enumeration wording is reproduced."""
+    """Every supported tool appears in the exact physical and turn-ending order."""
     template = load_operator_prompts()["main_template"]
     out = apply_operator_prompt_contract(template)
     assert (
-        "Physical device actions (`click`, `action_sequence`, `drag`, `input_text`, `swipe`, `press_key`,"
-        " `manage_app`, `wait_for_delay`, `long_press`)" in out
+        "Physical device actions (`click`, `action_sequence`, `drag`, `pinch`, `touch_gesture`,"
+        " `read_control`, `control_action`, `wait_until`, `scroll_to`, `input_text`, `swipe`,"
+        " `press_key`, `manage_app`, `wait_for_delay`, `long_press`)" in out
     )
     assert (
-        "Turn-Ending Action (`click`, `action_sequence`, `drag`, `swipe`, `input_text`, `long_press`,"
-        " `press_key`, `manage_app`, or `wait_for_delay`)" in out
+        "Turn-Ending Action (`click`, `action_sequence`, `drag`, `pinch`, `touch_gesture`,"
+        " `read_control`, `control_action`, `wait_until`, `scroll_to`, `swipe`, `input_text`,"
+        " `long_press`, `press_key`, `manage_app`, or `wait_for_delay`)" in out
     )
     assert "Helper/Subagent tools (`ask_explorer`, `ask_diagnoser`, `video_analyzer`)" in out
     assert "ADB/task tools (`run_adb_command`, `manage_task`, `android_file`)" in out