feat: prepare 2.2.0 with UI and execution fixes
Add dark theme, richer Markdown and readable plugin results. Improve Android input evidence, response integrity and diagnostics. Unify lifecycle, guidance receipts and evidence sources across clients. Keep direct action dispatch without an added AI quota approval layer. Align version metadata at 2.2.0 and bundle Helper 1.2.2. Validation: 3063 backend tests before the version bump; 44 after. Frontend: 317 tests and production build passed.
This commit is contained in:
1 parent
83bcfd54d5
commit
fac4ccb760
146 files changed
+5689
-2033
No files matched your search
@@ -12,6 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
|
||||
@@ -782,6 +783,33 @@ async def test_final_review_distinguishes_actual_delivery_from_report_claims():
|
||||
assert '"verified": false' in failed and "disk full" in failed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_final_review_receives_actual_source_ids_without_turning_backend_names_into_platforms():
|
||||
ctx = _mock_ctx()
|
||||
ctx.data_engine.get_agent_friendly_steps.return_value = [
|
||||
{
|
||||
"step_id": "real-step",
|
||||
"step_number": 7,
|
||||
"action_taken": {
|
||||
"plugin_id": "reader",
|
||||
"call_id": "real-call",
|
||||
"source_id": "real-source",
|
||||
"source": {"kind": "api", "target_id": "data-service", "label": "安卓业务后台"},
|
||||
},
|
||||
}
|
||||
]
|
||||
text = await _final_check_human_text(ctx)
|
||||
section = text.split(
|
||||
"# Recorded Source Inventory (provenance only, not verified assertions)\n", 1
|
||||
)[1].split("\n\n", 1)[0]
|
||||
value = json.loads(section)
|
||||
assert value["sources"][0]["kind"] == "api"
|
||||
assert value["sources"][0]["references"][0]["call_id"] == "real-call"
|
||||
guide = assemble_checker_prompt_segments("final", [], False)
|
||||
assert "does not by itself prove backend persistence" in guide
|
||||
assert "cannot silently overwrite the recorded result" in guide
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exploration_review_sees_conversation_and_can_read_earlier_messages():
|
||||
ctx = _mock_ctx()
|
||||
|
||||
@@ -72,6 +72,49 @@ def test_pc_first_and_intentional_interleaving_are_supported():
|
||||
assert progress.guard("cua-driver") is None
|
||||
|
||||
|
||||
def test_stage_updates_are_atomic_and_preserve_unmentioned_scope_and_order(tmp_path):
|
||||
progress = TaskProgress()
|
||||
resources = {"android", "cua-driver"}
|
||||
progress.update(plan(), resources=resources, directory=tmp_path)
|
||||
before = progress.value.model_dump()
|
||||
saved = (tmp_path / "notes/task_progress.md").read_bytes()
|
||||
with pytest.raises(ValidationError, match="完成或阻塞阶段必须记录"):
|
||||
progress.update(
|
||||
{"reason": "无证据不能完成", "stage_updates": [{"id": "phone", "status": "done"}]},
|
||||
resources=resources,
|
||||
directory=tmp_path,
|
||||
)
|
||||
assert progress.value.model_dump() == before
|
||||
assert (tmp_path / "notes/task_progress.md").read_bytes() == saved
|
||||
progress.update(
|
||||
{
|
||||
"reason": "手机已检查完",
|
||||
"stage_updates": [
|
||||
{"id": "phone", "status": "done", "evidence": "实际步骤记录 real-step"}
|
||||
],
|
||||
},
|
||||
resources=resources,
|
||||
directory=tmp_path,
|
||||
)
|
||||
assert progress.value.stages[1].model_dump() == before["stages"][1]
|
||||
assert [s.id for s in progress.value.stages] == ["phone", "pc"]
|
||||
assert progress.value.stages[0].scope == before["stages"][0]["scope"]
|
||||
progress.update(
|
||||
{"reason": "开始电脑检查", "stage_updates": [{"id": "pc", "status": "active"}]},
|
||||
resources=resources,
|
||||
)
|
||||
assert progress.guard("cua-driver") is None
|
||||
with pytest.raises(ValueError, match="未知 id"):
|
||||
progress.update(
|
||||
{"reason": "错拼不能覆盖", "stage_updates": [{"id": "typo", "status": "done"}]},
|
||||
resources=resources,
|
||||
)
|
||||
with pytest.raises(ValidationError, match="只能提供一种"):
|
||||
progress.update(
|
||||
{**plan(), "stage_updates": [{"id": "pc", "status": "active"}]}, resources=resources
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flash_preflight_does_not_authorize_pc_mutation(monkeypatch):
|
||||
plugins = SimpleNamespace(
|
||||
|
||||
@@ -19,6 +19,7 @@ from artemis.agents.operator.operator import OperatorNode
|
||||
from artemis.config.agent import MemoryTranscriptConfig
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.core.tool_failure import ToolFailure
|
||||
from artemis.utils.action_target import observation_anchor
|
||||
import pytest
|
||||
|
||||
# These tests exercise the legacy 2-message prompt path. Since M5 the
|
||||
@@ -126,6 +127,8 @@ async def test_perform_action_validation():
|
||||
"target_bounds": "[100,100][300,500]",
|
||||
"target_resource_id": "btn_1",
|
||||
"target_class": "android.widget.Button",
|
||||
"target_index": 1,
|
||||
**observation_anchor(mock_state),
|
||||
}
|
||||
]
|
||||
|
||||
@@ -156,6 +159,8 @@ async def test_perform_action_validation():
|
||||
"target_bounds": "[300,400][500,600]",
|
||||
"target_resource_id": "input_1",
|
||||
"target_class": "android.widget.EditText",
|
||||
"target_index": 2,
|
||||
**observation_anchor(mock_state),
|
||||
}
|
||||
|
||||
# Test swipe direction
|
||||
@@ -184,6 +189,8 @@ async def test_perform_action_validation():
|
||||
"target_bounds": "[300,400][500,600]",
|
||||
"target_resource_id": "input_1",
|
||||
"target_class": "android.widget.EditText",
|
||||
"target_index": 2,
|
||||
**observation_anchor(mock_state),
|
||||
}
|
||||
]
|
||||
|
||||
@@ -290,6 +297,7 @@ async def test_coordinate_click_requires_and_records_the_models_description():
|
||||
"times": 1,
|
||||
"delay_ms": 100,
|
||||
"target_description": "play button",
|
||||
**observation_anchor(mock_state),
|
||||
}
|
||||
]
|
||||
|
||||
@@ -883,6 +891,8 @@ async def test_long_press_action_translation():
|
||||
"target_bounds": "[100,100][300,500]",
|
||||
"target_resource_id": "btn_1",
|
||||
"target_class": "android.widget.Button",
|
||||
"target_index": 1,
|
||||
**observation_anchor(mock_state),
|
||||
}
|
||||
]
|
||||
|
||||
@@ -910,6 +920,7 @@ async def test_long_press_action_translation():
|
||||
"normalized_coordinates": [500, 600],
|
||||
"duration": 1000,
|
||||
"target_description": "app icon",
|
||||
**observation_anchor(mock_state),
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@@ -15,7 +15,12 @@
|
||||
"""Tests for pixel safety-net target descriptions."""
|
||||
|
||||
from pathlib import Path
|
||||
from io import BytesIO
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import AsyncMock
|
||||
import base64
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from artemis.agents.validator import precondition_pixel as pp
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
@@ -23,6 +28,58 @@ from langchain_core.messages import HumanMessage, SystemMessage
|
||||
_PNG = b"fake-image-bytes"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"response",
|
||||
[
|
||||
{"is_present": True, "confidence": 0.99, "target_kind": "surface", "target_matches": True},
|
||||
{"is_present": True, "confidence": 0.99},
|
||||
{"is_present": True, "confidence": 0.2, "target_kind": "control", "target_matches": True},
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"target",
|
||||
[
|
||||
{"action": "tap", "target_text": "Input"},
|
||||
{"action": "input_text", "target_description": "Input"},
|
||||
],
|
||||
)
|
||||
async def test_named_input_is_never_approved_as_blank_or_unverified(response, target, monkeypatch):
|
||||
monkeypatch.setattr(pp.visualization, "crop_and_annotate_target", lambda *a, **k: _PNG)
|
||||
monkeypatch.setattr(pp, "acomplete_structured", AsyncMock(return_value=response))
|
||||
session = AsyncMock()
|
||||
session.screenshot_b64.return_value = base64.b64encode(_PNG).decode()
|
||||
verdict = await pp._run_attempt(session, None, "prompt", _PNG, [1, 2], target, None, 1)
|
||||
assert verdict[0] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_renaming_rejected_point_cannot_dispatch_again(monkeypatch):
|
||||
monkeypatch.setattr(pp, "_init_llm_and_prompt", lambda *a: (None, "prompt"))
|
||||
attempt = AsyncMock(
|
||||
return_value=(False, pp.ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED, "No input")
|
||||
)
|
||||
monkeypatch.setattr(pp, "_run_attempt", attempt)
|
||||
ctx = MagicMock()
|
||||
image = Image.new("RGB", (400, 400), "white")
|
||||
frames = []
|
||||
for color in ("black", "blue"):
|
||||
image.putpixel((5, 5), Image.new("RGB", (1, 1), color).getpixel((0, 0)))
|
||||
encoded = BytesIO()
|
||||
image.save(encoded, format="PNG")
|
||||
frames.append(base64.b64encode(encoded.getvalue()).decode())
|
||||
for description, frame in zip(("pairing input", "blank surface"), frames, strict=True):
|
||||
verdict = await pp.validate_action_precondition_pixel(
|
||||
ctx,
|
||||
AsyncMock(),
|
||||
{"action": "tap", "coordinates": [200, 300], "target_description": description},
|
||||
frame,
|
||||
get_llm_fn=lambda: None,
|
||||
)
|
||||
assert verdict[0] is False
|
||||
assert attempt.await_count == 1
|
||||
|
||||
|
||||
def _texts(messages: list) -> list[str]:
|
||||
human = messages[1]
|
||||
return [part["text"] for part in human.content if part["type"] == "text"]
|
||||
|
||||
@@ -123,11 +123,19 @@ async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that ValidatorNode executes actions successfully."""
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
state.initial_goal = "在真实审批表单上 Approve once,不要普通聊天代答或重复提交。"
|
||||
|
||||
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
|
||||
with (
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
|
||||
patch(
|
||||
"artemis.services.llm.get_llm",
|
||||
side_effect=AssertionError("Unexpected AI action review"),
|
||||
) as get_llm,
|
||||
):
|
||||
node = ValidatorNode(mock_context)
|
||||
result = await node(state)
|
||||
|
||||
get_llm.assert_not_called()
|
||||
assert "last_execution_result" in result
|
||||
report = result["last_execution_result"]
|
||||
assert "execution" in report
|
||||
@@ -144,10 +152,40 @@ async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
|
||||
# tests/unit/mcp/test_action_session.py against the in-process ActionSession.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pro_unconfirmed_input_stops_burst_before_submission(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
mock_mcp.hierarchy = [
|
||||
{
|
||||
"class": "android.widget.EditText",
|
||||
"resource_id": "fixture:input",
|
||||
"text": "Paste here",
|
||||
"hint": "Paste here",
|
||||
"showing_hint": True,
|
||||
"focused": True,
|
||||
"bounds": "[100,200][500,400]",
|
||||
}
|
||||
]
|
||||
decisions = [
|
||||
{"action": "focus_and_input_text", "coordinates": [300, 300], "text": "fixture value"},
|
||||
{"action": "tap", "coordinates": [300, 500]},
|
||||
]
|
||||
state = DummyState(
|
||||
structured_decisions=json.dumps(decisions), latest_screenshot=temp_screenshot
|
||||
)
|
||||
with patch("artemis.utils.image_diff.check_ui_change", return_value=False):
|
||||
result = await ValidatorNode(mock_context)(state)
|
||||
report = result["last_execution_result"]
|
||||
assert report["status"] == "failed"
|
||||
assert report["execution"][0]["input_verification"] == "empty"
|
||||
assert len(mock_mcp.calls_for("input_text")) == 1
|
||||
assert not mock_mcp.calls_for("click")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_exec_error_opens_incident(mock_mcp, mock_context, temp_screenshot):
|
||||
"""A vetted single action that the device rejects is retried once, then an
|
||||
execution incident is opened for the Operator (no repair agent)."""
|
||||
"""An uncertain device failure opens an incident without replaying the write."""
|
||||
mock_mcp.action_handler = lambda name, args: (
|
||||
(False, "Error: Element not found") if name == "click" else (True, "")
|
||||
)
|
||||
@@ -165,8 +203,8 @@ async def test_validator_exec_error_opens_incident(mock_mcp, mock_context, temp_
|
||||
assert len(report["execution"]) == 1
|
||||
assert report["execution"][0]["attempts"] == [
|
||||
"Error: Element not found",
|
||||
"Error: Element not found",
|
||||
]
|
||||
assert len(mock_mcp.calls_for("click")) == 1
|
||||
assert "repair" not in report["execution"][0]
|
||||
|
||||
incident = report["incident"]
|
||||
|
||||
Reference in new issue
Block a user