feat: prepare 2.2.0 with UI and execution fixes

Add dark theme, richer Markdown and readable plugin results.
Improve Android input evidence, response integrity and diagnostics.
Unify lifecycle, guidance receipts and evidence sources across clients.
Keep direct action dispatch without an added AI quota approval layer.
Align version metadata at 2.2.0 and bundle Helper 1.2.2.

Validation: 3063 backend tests before the version bump; 44 after.
Frontend: 317 tests and production build passed.
This commit is contained in:
chenmingxuan committed 2026-10-26 00:44:13 +08:00
1 parent 83bcfd54d5
commit fac4ccb760
146 files changed
+5689 -2033

No files matched your search

+28
View File
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from unittest.mock import AsyncMock, MagicMock, patch
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
@@ -782,6 +783,33 @@ async def test_final_review_distinguishes_actual_delivery_from_report_claims():
assert '"verified": false' in failed and "disk full" in failed
@pytest.mark.asyncio
async def test_final_review_receives_actual_source_ids_without_turning_backend_names_into_platforms():
ctx = _mock_ctx()
ctx.data_engine.get_agent_friendly_steps.return_value = [
{
"step_id": "real-step",
"step_number": 7,
"action_taken": {
"plugin_id": "reader",
"call_id": "real-call",
"source_id": "real-source",
"source": {"kind": "api", "target_id": "data-service", "label": "安卓业务后台"},
},
}
]
text = await _final_check_human_text(ctx)
section = text.split(
"# Recorded Source Inventory (provenance only, not verified assertions)\n", 1
)[1].split("\n\n", 1)[0]
value = json.loads(section)
assert value["sources"][0]["kind"] == "api"
assert value["sources"][0]["references"][0]["call_id"] == "real-call"
guide = assemble_checker_prompt_segments("final", [], False)
assert "does not by itself prove backend persistence" in guide
assert "cannot silently overwrite the recorded result" in guide
@pytest.mark.asyncio
async def test_exploration_review_sees_conversation_and_can_read_earlier_messages():
ctx = _mock_ctx()
+43
View File
@@ -72,6 +72,49 @@ def test_pc_first_and_intentional_interleaving_are_supported():
assert progress.guard("cua-driver") is None
def test_stage_updates_are_atomic_and_preserve_unmentioned_scope_and_order(tmp_path):
progress = TaskProgress()
resources = {"android", "cua-driver"}
progress.update(plan(), resources=resources, directory=tmp_path)
before = progress.value.model_dump()
saved = (tmp_path / "notes/task_progress.md").read_bytes()
with pytest.raises(ValidationError, match="完成或阻塞阶段必须记录"):
progress.update(
{"reason": "无证据不能完成", "stage_updates": [{"id": "phone", "status": "done"}]},
resources=resources,
directory=tmp_path,
)
assert progress.value.model_dump() == before
assert (tmp_path / "notes/task_progress.md").read_bytes() == saved
progress.update(
{
"reason": "手机已检查完",
"stage_updates": [
{"id": "phone", "status": "done", "evidence": "实际步骤记录 real-step"}
],
},
resources=resources,
directory=tmp_path,
)
assert progress.value.stages[1].model_dump() == before["stages"][1]
assert [s.id for s in progress.value.stages] == ["phone", "pc"]
assert progress.value.stages[0].scope == before["stages"][0]["scope"]
progress.update(
{"reason": "开始电脑检查", "stage_updates": [{"id": "pc", "status": "active"}]},
resources=resources,
)
assert progress.guard("cua-driver") is None
with pytest.raises(ValueError, match="未知 id"):
progress.update(
{"reason": "错拼不能覆盖", "stage_updates": [{"id": "typo", "status": "done"}]},
resources=resources,
)
with pytest.raises(ValidationError, match="只能提供一种"):
progress.update(
{**plan(), "stage_updates": [{"id": "pc", "status": "active"}]}, resources=resources
)
@pytest.mark.asyncio
async def test_flash_preflight_does_not_authorize_pc_mutation(monkeypatch):
plugins = SimpleNamespace(
+11
View File
@@ -19,6 +19,7 @@ from artemis.agents.operator.operator import OperatorNode
from artemis.config.agent import MemoryTranscriptConfig
from artemis.context import ArtemisContext
from artemis.core.tool_failure import ToolFailure
from artemis.utils.action_target import observation_anchor
import pytest
# These tests exercise the legacy 2-message prompt path. Since M5 the
@@ -126,6 +127,8 @@ async def test_perform_action_validation():
"target_bounds": "[100,100][300,500]",
"target_resource_id": "btn_1",
"target_class": "android.widget.Button",
"target_index": 1,
**observation_anchor(mock_state),
}
]
@@ -156,6 +159,8 @@ async def test_perform_action_validation():
"target_bounds": "[300,400][500,600]",
"target_resource_id": "input_1",
"target_class": "android.widget.EditText",
"target_index": 2,
**observation_anchor(mock_state),
}
# Test swipe direction
@@ -184,6 +189,8 @@ async def test_perform_action_validation():
"target_bounds": "[300,400][500,600]",
"target_resource_id": "input_1",
"target_class": "android.widget.EditText",
"target_index": 2,
**observation_anchor(mock_state),
}
]
@@ -290,6 +297,7 @@ async def test_coordinate_click_requires_and_records_the_models_description():
"times": 1,
"delay_ms": 100,
"target_description": "play button",
**observation_anchor(mock_state),
}
]
@@ -883,6 +891,8 @@ async def test_long_press_action_translation():
"target_bounds": "[100,100][300,500]",
"target_resource_id": "btn_1",
"target_class": "android.widget.Button",
"target_index": 1,
**observation_anchor(mock_state),
}
]
@@ -910,6 +920,7 @@ async def test_long_press_action_translation():
"normalized_coordinates": [500, 600],
"duration": 1000,
"target_description": "app icon",
**observation_anchor(mock_state),
}
]
@@ -15,7 +15,12 @@
"""Tests for pixel safety-net target descriptions."""
from pathlib import Path
from io import BytesIO
from unittest.mock import MagicMock
from unittest.mock import AsyncMock
import base64
import pytest
from PIL import Image
from artemis.agents.validator import precondition_pixel as pp
from langchain_core.messages import HumanMessage, SystemMessage
@@ -23,6 +28,58 @@ from langchain_core.messages import HumanMessage, SystemMessage
_PNG = b"fake-image-bytes"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"response",
[
{"is_present": True, "confidence": 0.99, "target_kind": "surface", "target_matches": True},
{"is_present": True, "confidence": 0.99},
{"is_present": True, "confidence": 0.2, "target_kind": "control", "target_matches": True},
],
)
@pytest.mark.parametrize(
"target",
[
{"action": "tap", "target_text": "Input"},
{"action": "input_text", "target_description": "Input"},
],
)
async def test_named_input_is_never_approved_as_blank_or_unverified(response, target, monkeypatch):
monkeypatch.setattr(pp.visualization, "crop_and_annotate_target", lambda *a, **k: _PNG)
monkeypatch.setattr(pp, "acomplete_structured", AsyncMock(return_value=response))
session = AsyncMock()
session.screenshot_b64.return_value = base64.b64encode(_PNG).decode()
verdict = await pp._run_attempt(session, None, "prompt", _PNG, [1, 2], target, None, 1)
assert verdict[0] is False
@pytest.mark.asyncio
async def test_renaming_rejected_point_cannot_dispatch_again(monkeypatch):
monkeypatch.setattr(pp, "_init_llm_and_prompt", lambda *a: (None, "prompt"))
attempt = AsyncMock(
return_value=(False, pp.ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED, "No input")
)
monkeypatch.setattr(pp, "_run_attempt", attempt)
ctx = MagicMock()
image = Image.new("RGB", (400, 400), "white")
frames = []
for color in ("black", "blue"):
image.putpixel((5, 5), Image.new("RGB", (1, 1), color).getpixel((0, 0)))
encoded = BytesIO()
image.save(encoded, format="PNG")
frames.append(base64.b64encode(encoded.getvalue()).decode())
for description, frame in zip(("pairing input", "blank surface"), frames, strict=True):
verdict = await pp.validate_action_precondition_pixel(
ctx,
AsyncMock(),
{"action": "tap", "coordinates": [200, 300], "target_description": description},
frame,
get_llm_fn=lambda: None,
)
assert verdict[0] is False
assert attempt.await_count == 1
def _texts(messages: list) -> list[str]:
human = messages[1]
return [part["text"] for part in human.content if part["type"] == "text"]
+42 -4
View File
@@ -123,11 +123,19 @@ async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
"""Test that ValidatorNode executes actions successfully."""
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
state.initial_goal = "在真实审批表单上 Approve once,不要普通聊天代答或重复提交。"
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
with (
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch(
"artemis.services.llm.get_llm",
side_effect=AssertionError("Unexpected AI action review"),
) as get_llm,
):
node = ValidatorNode(mock_context)
result = await node(state)
get_llm.assert_not_called()
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
@@ -144,10 +152,40 @@ async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
# tests/unit/mcp/test_action_session.py against the in-process ActionSession.
@pytest.mark.asyncio
async def test_pro_unconfirmed_input_stops_burst_before_submission(
mock_mcp, mock_context, temp_screenshot
):
mock_mcp.hierarchy = [
{
"class": "android.widget.EditText",
"resource_id": "fixture:input",
"text": "Paste here",
"hint": "Paste here",
"showing_hint": True,
"focused": True,
"bounds": "[100,200][500,400]",
}
]
decisions = [
{"action": "focus_and_input_text", "coordinates": [300, 300], "text": "fixture value"},
{"action": "tap", "coordinates": [300, 500]},
]
state = DummyState(
structured_decisions=json.dumps(decisions), latest_screenshot=temp_screenshot
)
with patch("artemis.utils.image_diff.check_ui_change", return_value=False):
result = await ValidatorNode(mock_context)(state)
report = result["last_execution_result"]
assert report["status"] == "failed"
assert report["execution"][0]["input_verification"] == "empty"
assert len(mock_mcp.calls_for("input_text")) == 1
assert not mock_mcp.calls_for("click")
@pytest.mark.asyncio
async def test_validator_exec_error_opens_incident(mock_mcp, mock_context, temp_screenshot):
"""A vetted single action that the device rejects is retried once, then an
execution incident is opened for the Operator (no repair agent)."""
"""An uncertain device failure opens an incident without replaying the write."""
mock_mcp.action_handler = lambda name, args: (
(False, "Error: Element not found") if name == "click" else (True, "")
)
@@ -165,8 +203,8 @@ async def test_validator_exec_error_opens_incident(mock_mcp, mock_context, temp_
assert len(report["execution"]) == 1
assert report["execution"][0]["attempts"] == [
"Error: Element not found",
"Error: Element not found",
]
assert len(mock_mcp.calls_for("click")) == 1
assert "repair" not in report["execution"][0]
incident = report["incident"]