feat: prepare 2.2.0 with UI and execution fixes

Add dark theme, richer Markdown and readable plugin results.
Improve Android input evidence, response integrity and diagnostics.
Unify lifecycle, guidance receipts and evidence sources across clients.
Keep direct action dispatch without an added AI quota approval layer.
Align version metadata at 2.2.0 and bundle Helper 1.2.2.

Validation: 3063 backend tests before the version bump; 44 after.
Frontend: 317 tests and production build passed.
This commit is contained in:
chenmingxuan committed 2026-10-26 00:44:13 +08:00
1 parent 83bcfd54d5
commit fac4ccb760
146 files changed
+5689 -2033

No files matched your search

+107
View File
@@ -1173,6 +1173,112 @@ def test_android_artifact_export_does_not_follow_other_session_links(isolated):
assert reports.session_artifacts({"id": "session"}) == []
def test_report_source_inventory_preserves_human_verdict_and_raw_evidence(
isolated, tmp_path, monkeypatch
):
from apps.admin_console.services import evidence_service, task_inspection
job = isolated.jobs(service.create_batch(payload())["id"])[0]
sid = job["session_id"]
action = {
"action": "plug_read",
"plugin_id": "data-reader",
"call_id": "recorded-call",
"read_only": True,
"source": {"kind": "api", "target_id": "business-service", "label": "安卓业务后台"},
}
with sqlite3.connect(isolated.path) as conn:
conn.execute(
"INSERT INTO sessions(session_id,initial_goal,start_time,end_time,status) VALUES (?,?,?,?,?)",
(sid, "检查业务状态", 1700000010, 1700000120, "completed"),
)
conn.execute(
"INSERT INTO steps(step_id,session_id,step_number,action_taken) VALUES (?,?,?,?)",
("actual-step", sid, 7, json.dumps(action)),
)
isolated.save_case_review(sid, "failed", "用户指出业务状态仍未落库")
monkeypatch.setattr(evidence_service, "TRACES_PATH", reports.TRACES_PATH)
report = reports.session_report(sid)
source = report["evidence_sources"]["sources"][0]
assert source["category"] == "auxiliary"
assert source["references"][0]["step_id"] == "actual-step"
assert report["test_result"]["status"] == "failed"
assert report["test_result"]["source"] == "human"
assert report["steps"][0]["action_taken"] == action
index = evidence_service.export_index(sid, report, [])
assert index["evidence_sources"] == report["evidence_sources"]
monkeypatch.setattr(task_inspection, "store", isolated)
monkeypatch.setattr(task_inspection, "session_repo", reports.SessionRepository())
assert (
task_inspection.inspect_task(sid, "report")["evidence_sources"]
== report["evidence_sources"]
)
from docx import Document
from openpyxl import load_workbook
word = tmp_path / "sources.docx"
sheet = tmp_path / "sources.xlsx"
reports.word_report(report, word)
reports.excel_report(report, sheet)
document = Document(word)
word_text = " ".join(
[p.text for p in document.paragraphs]
+ [cell.text for table in document.tables for row in table.rows for cell in row.cells]
)
workbook = load_workbook(sheet)
try:
excel_text = " ".join(
str(cell.value or "") for ws in workbook for row in ws for cell in row
)
finally:
workbook.close()
for text in (word_text, excel_text):
assert "接口数据" in text
assert "安卓业务后台" in text
assert "actual-step" in text
assert "用户指出业务状态仍未落库" in text
def test_reports_share_lifecycle_and_distinguish_guidance_receipt_from_consumption(
isolated, tmp_path, monkeypatch
):
from apps.admin_console.services import task_lifecycle
from artemis.runtime import task_control
job = isolated.jobs(service.create_batch(payload())["id"])[0]
job.update(
status="completed",
created_at=1700000010,
worker_started_at=1700000020,
ended_at=1700000120,
finalized_at=1700000125,
)
isolated.save_job(job)
monkeypatch.setattr(task_control, "TRACES_PATH", reports.TRACES_PATH)
task_control.add_instruction(job["session_id"], "保留证据", instruction_id="unused-guidance")
report = reports.session_report(job["session_id"])
assert report["lifecycle"] == task_lifecycle.task_lifecycle(
job["session_id"], db_path=isolated.path
)
assert report["lifecycle"]["durations"]["total"] == 115
assert report["lifecycle"]["durations"]["execution"] is None
assert report["instructions"][0]["status"] == "unapplied"
word = tmp_path / "lifecycle.docx"
excel = tmp_path / "lifecycle.xlsx"
reports.word_report(report, word)
reports.excel_report(report, excel)
document = Document(word)
word_text = "\n".join(p.text for p in document.paragraphs) + "\n".join(
cell.text for table in document.tables for row in table.rows for cell in row.cells
)
book = load_workbook(excel)
excel_text = "\n".join(str(cell.value) for sheet in book for row in sheet for cell in row)
for content in (word_text, excel_text):
assert "后台排队:10.0 秒" in content
assert "执行与整理:未记录" in content
assert "unused-guidance · 任务结束,未记录消费 · 接收" in content
def test_video_export_accepts_finalized_worker_trace_and_rejects_external_path(
isolated, tmp_path, monkeypatch
):
@@ -1188,6 +1294,7 @@ def test_video_export_accepts_finalized_worker_trace_and_rejects_external_path(
second.write_bytes(b"second segment")
repository = Mock()
repository.db_path = isolated.path
repository.get_video_recording_for_session.return_value = {"local_video_path": str(first)}
repository.get_session_by_id.return_value = {"video_filepath": str(first)}
monkeypatch.setattr(reports, "SessionRepository", lambda: repository)
@@ -99,6 +99,7 @@ async def test_stop_requests_graceful_cancel_and_skips_kill_when_worker_exits():
# The worker is asked to cancel itself; nothing is killed yet.
terminate.assert_not_called()
assert cancel_requests.is_cancel_requested(session_id="sess-1")
assert cancel_requests.get_cancel_reason(session_id="sess-1").value == "user_stop"
assert cancel_requests.is_cancel_requested(pid=24680, process_created_at=1234.5)
assert len(TaskQueueService._forced_stop_tasks) == 1
enforcer = next(iter(TaskQueueService._forced_stop_tasks))
@@ -180,6 +180,37 @@ def test_old_checker_config_does_not_enable_flash_review(monkeypatch, tmp_path):
assert load_agent_config(path).flash.final_check_enabled is False
@pytest.mark.parametrize("enabled", [True, False])
def test_saved_flash_switch_reaches_next_worker_without_restarting(
isolated_settings, monkeypatch, enabled
):
from apps.admin_console.services.execution_config import execution_config
from apps.admin_console.services.task_queue_service import TaskQueueService
monkeypatch.setattr("artemis.exploration.store.explorations.run", lambda _sid: None)
monkeypatch.setattr("apps.admin_console.services.automation_store.store.job", lambda _sid: None)
monkeypatch.setenv("ARTEMIS_FLASH_FINAL_CHECK_ENABLED", str(not enabled).lower())
running = {"profile": "flash", "goal": "already running", "session_id": "running-review"}
running["execution_config"] = execution_config(running)
queued = {"profile": "flash", "goal": "queued", "session_id": "queued-review"}
queued["execution_config"] = execution_config(queued)
service.save_capability_settings(service.CapabilitySettings(flash_final_check_enabled=enabled))
# The real queue refreshes pending snapshots just before starting their worker.
queued["execution_config"] = execution_config(queued)
for item, expected in ((running, not enabled), (queued, enabled)):
_cmd, env = TaskQueueService._build_worker_invocation(
item,
item["session_id"],
item["session_id"],
item["goal"],
"flash",
TaskQueueService._task_target(item),
)
assert env["ARTEMIS_FLASH_FINAL_CHECK_ENABLED"] == str(expected).lower()
assert load_agent_config().pro.checker.final_check is True
def test_screenshot_json_can_override_environment(monkeypatch, tmp_path):
config_path = tmp_path / "artemis.jsonc"
config_path.write_text(
@@ -0,0 +1,115 @@
"""Shared task progress stays read-only and distinguishes receipt from consumption."""
import json
import sqlite3
from fastapi import FastAPI
from fastapi.testclient import TestClient
import pytest
from apps.admin_console.routers import tasks
from apps.admin_console.services import task_lifecycle as service
from artemis.runtime import task_control as control
@pytest.fixture
def recorded(tmp_path, monkeypatch):
db = tmp_path / "只读 progress.db"
sid = "recorded-task"
with sqlite3.connect(db) as conn:
conn.executescript(
"CREATE TABLE sessions (session_id TEXT, status TEXT, pid INTEGER, "
"start_time REAL, end_time REAL);"
"CREATE TABLE automation_jobs (session_id TEXT, payload TEXT);"
"CREATE TABLE traces (session_id TEXT, type TEXT, payload TEXT);"
)
conn.execute("INSERT INTO sessions VALUES (?, 'completed', 17, 20, 120)", (sid,))
conn.execute(
"INSERT INTO automation_jobs VALUES (?, ?)",
(
sid,
json.dumps(
{
"status": "completed",
"created_at": 10,
"worker_started_at": 20,
"ended_at": 120,
"finalized_at": 125,
}
),
),
)
conn.execute(
"INSERT INTO traces VALUES (?, 'final_review_diagnostic', ?)",
(sid, json.dumps({"started_at": 90, "ended_at": 110, "status": "completed"})),
)
monkeypatch.setattr(control, "TRACES_PATH", tmp_path / "traces")
monkeypatch.setattr(service, "DB_PATH", db)
control._write_atomic(
control.control_dir(sid) / "task_control_state.json",
{
"pid": 17,
"status": "running",
"timing": {
"started_at": 20,
"state_started_at": 80,
"seconds": {"queued": 10, "task_paused": 20, "running": 30},
},
},
)
return db, sid
def test_finished_projection_is_stable_and_does_not_bootstrap_or_change_database(recorded):
db, sid = recorded
before = db.read_bytes()
first = service.task_lifecycle(sid, db_path=str(db), now=500)
assert first == service.task_lifecycle(sid, db_path=db, now=900)
assert first["durations"] == {
"total": 115,
"queued": 10,
"resource_wait": 10,
"paused": 20,
"final_review": 20,
"execution": 50,
"report_export": 5,
}
assert db.read_bytes() == before
def test_legacy_guidance_array_and_detailed_web_receipt_share_terminal_state(recorded):
_, sid = recorded
pending = control.add_instruction(sid, "保存现有证据", instruction_id="pending")
original = control.control_dir(sid) / "instructions" / "pending.json"
before = original.read_bytes()
app = FastAPI()
app.include_router(tasks.router)
client = TestClient(app)
details = client.get(f"/api/tasks/{sid}/guidance?details=true").json()
legacy = client.get(f"/api/tasks/{sid}/guidance").json()
assert details["instructions"] == legacy
assert legacy[0]["status"] == "unapplied"
assert legacy[0]["accepted_at"] == pending["timestamp"]
assert details["lifecycle"]["guidance_counts"]["unapplied"] == 1
assert original.read_bytes() == before
def test_state_from_another_worker_does_not_supply_current_timing(recorded):
db, sid = recorded
state = control.read_state(sid)
state["pid"] = 18
control._write_atomic(control.control_dir(sid) / "task_control_state.json", state)
value = service.task_lifecycle(sid, db_path=db)
assert not value["control_timing_recorded"]
assert value["durations"]["execution"] is None
assert value["durations"]["resource_wait"] is None
def test_missing_database_is_not_created_and_does_not_hide_guidance(tmp_path, monkeypatch):
monkeypatch.setattr(control, "TRACES_PATH", tmp_path)
control.add_instruction("new-task", "等待安全检查点")
path = tmp_path / "missing.db"
value = service.task_lifecycle("new-task", db_path=path, now=100)
assert value["phase"] == "unknown"
assert value["guidance_counts"]["pending"] == 1
assert not path.exists()
+28
View File
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from unittest.mock import AsyncMock, MagicMock, patch
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
@@ -782,6 +783,33 @@ async def test_final_review_distinguishes_actual_delivery_from_report_claims():
assert '"verified": false' in failed and "disk full" in failed
@pytest.mark.asyncio
async def test_final_review_receives_actual_source_ids_without_turning_backend_names_into_platforms():
ctx = _mock_ctx()
ctx.data_engine.get_agent_friendly_steps.return_value = [
{
"step_id": "real-step",
"step_number": 7,
"action_taken": {
"plugin_id": "reader",
"call_id": "real-call",
"source_id": "real-source",
"source": {"kind": "api", "target_id": "data-service", "label": "安卓业务后台"},
},
}
]
text = await _final_check_human_text(ctx)
section = text.split(
"# Recorded Source Inventory (provenance only, not verified assertions)\n", 1
)[1].split("\n\n", 1)[0]
value = json.loads(section)
assert value["sources"][0]["kind"] == "api"
assert value["sources"][0]["references"][0]["call_id"] == "real-call"
guide = assemble_checker_prompt_segments("final", [], False)
assert "does not by itself prove backend persistence" in guide
assert "cannot silently overwrite the recorded result" in guide
@pytest.mark.asyncio
async def test_exploration_review_sees_conversation_and_can_read_earlier_messages():
ctx = _mock_ctx()
+43
View File
@@ -72,6 +72,49 @@ def test_pc_first_and_intentional_interleaving_are_supported():
assert progress.guard("cua-driver") is None
def test_stage_updates_are_atomic_and_preserve_unmentioned_scope_and_order(tmp_path):
progress = TaskProgress()
resources = {"android", "cua-driver"}
progress.update(plan(), resources=resources, directory=tmp_path)
before = progress.value.model_dump()
saved = (tmp_path / "notes/task_progress.md").read_bytes()
with pytest.raises(ValidationError, match="完成或阻塞阶段必须记录"):
progress.update(
{"reason": "无证据不能完成", "stage_updates": [{"id": "phone", "status": "done"}]},
resources=resources,
directory=tmp_path,
)
assert progress.value.model_dump() == before
assert (tmp_path / "notes/task_progress.md").read_bytes() == saved
progress.update(
{
"reason": "手机已检查完",
"stage_updates": [
{"id": "phone", "status": "done", "evidence": "实际步骤记录 real-step"}
],
},
resources=resources,
directory=tmp_path,
)
assert progress.value.stages[1].model_dump() == before["stages"][1]
assert [s.id for s in progress.value.stages] == ["phone", "pc"]
assert progress.value.stages[0].scope == before["stages"][0]["scope"]
progress.update(
{"reason": "开始电脑检查", "stage_updates": [{"id": "pc", "status": "active"}]},
resources=resources,
)
assert progress.guard("cua-driver") is None
with pytest.raises(ValueError, match="未知 id"):
progress.update(
{"reason": "错拼不能覆盖", "stage_updates": [{"id": "typo", "status": "done"}]},
resources=resources,
)
with pytest.raises(ValidationError, match="只能提供一种"):
progress.update(
{**plan(), "stage_updates": [{"id": "pc", "status": "active"}]}, resources=resources
)
@pytest.mark.asyncio
async def test_flash_preflight_does_not_authorize_pc_mutation(monkeypatch):
plugins = SimpleNamespace(
+11
View File
@@ -19,6 +19,7 @@ from artemis.agents.operator.operator import OperatorNode
from artemis.config.agent import MemoryTranscriptConfig
from artemis.context import ArtemisContext
from artemis.core.tool_failure import ToolFailure
from artemis.utils.action_target import observation_anchor
import pytest
# These tests exercise the legacy 2-message prompt path. Since M5 the
@@ -126,6 +127,8 @@ async def test_perform_action_validation():
"target_bounds": "[100,100][300,500]",
"target_resource_id": "btn_1",
"target_class": "android.widget.Button",
"target_index": 1,
**observation_anchor(mock_state),
}
]
@@ -156,6 +159,8 @@ async def test_perform_action_validation():
"target_bounds": "[300,400][500,600]",
"target_resource_id": "input_1",
"target_class": "android.widget.EditText",
"target_index": 2,
**observation_anchor(mock_state),
}
# Test swipe direction
@@ -184,6 +189,8 @@ async def test_perform_action_validation():
"target_bounds": "[300,400][500,600]",
"target_resource_id": "input_1",
"target_class": "android.widget.EditText",
"target_index": 2,
**observation_anchor(mock_state),
}
]
@@ -290,6 +297,7 @@ async def test_coordinate_click_requires_and_records_the_models_description():
"times": 1,
"delay_ms": 100,
"target_description": "play button",
**observation_anchor(mock_state),
}
]
@@ -883,6 +891,8 @@ async def test_long_press_action_translation():
"target_bounds": "[100,100][300,500]",
"target_resource_id": "btn_1",
"target_class": "android.widget.Button",
"target_index": 1,
**observation_anchor(mock_state),
}
]
@@ -910,6 +920,7 @@ async def test_long_press_action_translation():
"normalized_coordinates": [500, 600],
"duration": 1000,
"target_description": "app icon",
**observation_anchor(mock_state),
}
]
@@ -15,7 +15,12 @@
"""Tests for pixel safety-net target descriptions."""
from pathlib import Path
from io import BytesIO
from unittest.mock import MagicMock
from unittest.mock import AsyncMock
import base64
import pytest
from PIL import Image
from artemis.agents.validator import precondition_pixel as pp
from langchain_core.messages import HumanMessage, SystemMessage
@@ -23,6 +28,58 @@ from langchain_core.messages import HumanMessage, SystemMessage
_PNG = b"fake-image-bytes"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"response",
[
{"is_present": True, "confidence": 0.99, "target_kind": "surface", "target_matches": True},
{"is_present": True, "confidence": 0.99},
{"is_present": True, "confidence": 0.2, "target_kind": "control", "target_matches": True},
],
)
@pytest.mark.parametrize(
"target",
[
{"action": "tap", "target_text": "Input"},
{"action": "input_text", "target_description": "Input"},
],
)
async def test_named_input_is_never_approved_as_blank_or_unverified(response, target, monkeypatch):
monkeypatch.setattr(pp.visualization, "crop_and_annotate_target", lambda *a, **k: _PNG)
monkeypatch.setattr(pp, "acomplete_structured", AsyncMock(return_value=response))
session = AsyncMock()
session.screenshot_b64.return_value = base64.b64encode(_PNG).decode()
verdict = await pp._run_attempt(session, None, "prompt", _PNG, [1, 2], target, None, 1)
assert verdict[0] is False
@pytest.mark.asyncio
async def test_renaming_rejected_point_cannot_dispatch_again(monkeypatch):
monkeypatch.setattr(pp, "_init_llm_and_prompt", lambda *a: (None, "prompt"))
attempt = AsyncMock(
return_value=(False, pp.ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED, "No input")
)
monkeypatch.setattr(pp, "_run_attempt", attempt)
ctx = MagicMock()
image = Image.new("RGB", (400, 400), "white")
frames = []
for color in ("black", "blue"):
image.putpixel((5, 5), Image.new("RGB", (1, 1), color).getpixel((0, 0)))
encoded = BytesIO()
image.save(encoded, format="PNG")
frames.append(base64.b64encode(encoded.getvalue()).decode())
for description, frame in zip(("pairing input", "blank surface"), frames, strict=True):
verdict = await pp.validate_action_precondition_pixel(
ctx,
AsyncMock(),
{"action": "tap", "coordinates": [200, 300], "target_description": description},
frame,
get_llm_fn=lambda: None,
)
assert verdict[0] is False
assert attempt.await_count == 1
def _texts(messages: list) -> list[str]:
human = messages[1]
return [part["text"] for part in human.content if part["type"] == "text"]
+42 -4
View File
@@ -123,11 +123,19 @@ async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
"""Test that ValidatorNode executes actions successfully."""
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
state.initial_goal = "在真实审批表单上 Approve once,不要普通聊天代答或重复提交。"
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
with (
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch(
"artemis.services.llm.get_llm",
side_effect=AssertionError("Unexpected AI action review"),
) as get_llm,
):
node = ValidatorNode(mock_context)
result = await node(state)
get_llm.assert_not_called()
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
@@ -144,10 +152,40 @@ async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
# tests/unit/mcp/test_action_session.py against the in-process ActionSession.
@pytest.mark.asyncio
async def test_pro_unconfirmed_input_stops_burst_before_submission(
mock_mcp, mock_context, temp_screenshot
):
mock_mcp.hierarchy = [
{
"class": "android.widget.EditText",
"resource_id": "fixture:input",
"text": "Paste here",
"hint": "Paste here",
"showing_hint": True,
"focused": True,
"bounds": "[100,200][500,400]",
}
]
decisions = [
{"action": "focus_and_input_text", "coordinates": [300, 300], "text": "fixture value"},
{"action": "tap", "coordinates": [300, 500]},
]
state = DummyState(
structured_decisions=json.dumps(decisions), latest_screenshot=temp_screenshot
)
with patch("artemis.utils.image_diff.check_ui_change", return_value=False):
result = await ValidatorNode(mock_context)(state)
report = result["last_execution_result"]
assert report["status"] == "failed"
assert report["execution"][0]["input_verification"] == "empty"
assert len(mock_mcp.calls_for("input_text")) == 1
assert not mock_mcp.calls_for("click")
@pytest.mark.asyncio
async def test_validator_exec_error_opens_incident(mock_mcp, mock_context, temp_screenshot):
"""A vetted single action that the device rejects is retried once, then an
execution incident is opened for the Operator (no repair agent)."""
"""An uncertain device failure opens an incident without replaying the write."""
mock_mcp.action_handler = lambda name, args: (
(False, "Error: Element not found") if name == "click" else (True, "")
)
@@ -165,8 +203,8 @@ async def test_validator_exec_error_opens_incident(mock_mcp, mock_context, temp_
assert len(report["execution"]) == 1
assert report["execution"][0]["attempts"] == [
"Error: Element not found",
"Error: Element not found",
]
assert len(mock_mcp.calls_for("click")) == 1
assert "repair" not in report["execution"][0]
incident = report["incident"]
@@ -0,0 +1,105 @@
from copy import deepcopy
from artemis.data_engine.evidence_sources import recorded_sources
def record(kind, target, **extra):
return {
"plugin_id": "adapter",
"source": {"kind": kind, "target_id": target, "label": "安卓业务后台"},
**extra,
}
def test_business_backend_name_does_not_become_an_execution_platform_or_assertion():
steps = [
{
"step_id": "step-real-id",
"step_number": 7,
"last_execution_result": {
"execution": [record("api", "backend", call_id="call-real-id", read_only=True)]
},
}
]
before = deepcopy(steps)
value = recorded_sources(steps)
source = value["sources"][0]
assert source["kind"] == "api"
assert source["category"] == "auxiliary"
assert source["references"] == [
{"step_id": "step-real-id", "step_number": 7, "call_id": "call-real-id", "read_only": True}
]
assert not any(key in source for key in ("passed", "verified", "verdict"))
assert steps == before
def test_sources_remain_separate_and_references_use_host_ids():
steps = [
{
"step_id": "s1",
"step_number": 1,
"action_taken": record("computer", "window", call_id="c1"),
},
{
"step_id": "s2",
"step_number": 2,
"action_taken": record("computer", "window", call_id="c2"),
},
{
"step_id": "s3",
"step_number": 3,
"action_taken": record("android", "phone", call_id="c3"),
},
]
value = recorded_sources(steps)
assert value["total_sources"] == 2
assert value["sources"][0]["reference_count"] == 2
assert [r["call_id"] for r in value["sources"][0]["references"]] == ["c1", "c2"]
assert value["sources"][0]["id"] == recorded_sources(steps[:1])["sources"][0]["id"]
assert value["sources"][1]["category"] == "interface"
def test_model_arguments_and_unsourced_images_do_not_fabricate_source_metadata():
value = recorded_sources(
[
{
"step_id": "s",
"step_number": 3,
"pre_image_name": "hash",
"action_taken": {"args": {"source": {"kind": "computer", "target_id": "claim"}}},
}
]
)
assert value["sources"] == []
assert value["unattributed_references"] == [{"step_id": "s", "step_number": 3}]
def test_summary_is_bounded_and_reports_omissions_explicitly():
steps = [{"step_id": str(i), "action_taken": record("file", str(i))} for i in range(10)]
value = recorded_sources(steps, limit=2)
assert len(value["sources"]) == 2
assert value["total_sources"] == 10
assert value["sources_omitted"] == 8
repeated = recorded_sources(
[{"step_id": str(i), "action_taken": record("api", "one")} for i in range(20)]
)
assert len(repeated["sources"][0]["references"]) == 8
assert repeated["sources"][0]["reference_count"] == 20
def test_partial_source_metadata_does_not_hide_unknown_actions_in_the_same_step():
value = recorded_sources(
[
{
"step_id": "mixed",
"last_execution_result": {
"execution": [
record("api", "service"),
{"action": "tap"},
]
},
}
]
)
assert value["total_sources"] == 1
assert value["unattributed_step_count"] == 1
@@ -8,11 +8,13 @@ action), and report their status explicitly instead of sniffing the text for
the word "Error".
"""
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock, patch
import pytest
from artemis.mcp.action_executor import AGENT_TOOL_NAMES, McpActionExecutor
from artemis.mcp.action_types import ActionCode, ActionResult
from artemis.tools.explorer_tool import ExplorerCandidate, ExplorerOutcome
@@ -48,6 +50,61 @@ def test_helper_tools_are_not_device_actions():
assert not (AGENT_TOOL_NAMES & executor.action_tool_names)
@pytest.mark.asyncio
@pytest.mark.parametrize("extension", [False, True])
@pytest.mark.parametrize("ok", [False, True])
async def test_authorized_actions_dispatch_without_secondary_ai_review(extension, ok):
from mcp.types import CallToolResult, TextContent
executor = _make_executor(agent_name="flash")
state = _state()
state.initial_goal = (
"仅发送一次指定的问题。等待真实审批表单,Approve once (not remember rule)。"
"不要普通聊天代答或重复提交;不向该测试另发消息、不停止任务,也不操作其他会话。"
)
name = "submit_form" if extension else "click"
args = (
{"window_id": 123}
if extension
else {
"target": [500, 500],
"target_description": "Approve",
}
)
executor.actuator.extensions.return_value = [SimpleNamespace(name="submit_form")]
session = SimpleNamespace(
call=AsyncMock(
return_value=ActionResult(
ok=ok,
code=ActionCode.OK if ok else ActionCode.DEVICE_ERROR,
action=name,
message="tool outcome",
)
),
call_raw=AsyncMock(
return_value=CallToolResult(
isError=not ok, content=[TextContent(type="text", text="tool outcome")]
)
),
)
executor._session_or_start = AsyncMock(return_value=session)
executor._observe_after_action = AsyncMock(return_value=(SimpleNamespace(ok=False), None))
with patch(
"artemis.services.llm.get_llm", side_effect=AssertionError("Unexpected AI action review")
) as get_llm:
result = await executor.execute(name, args, "action-1", state)
get_llm.assert_not_called()
wire_args = args if extension else {"target": [500, 500], "times": 1, "delay_ms": 100}
dispatched = session.call_raw if extension else session.call
unused = session.call if extension else session.call_raw
dispatched.assert_awaited_once_with(name, wire_args)
unused.assert_not_awaited()
assert result.status == ("success" if ok else "error")
assert result.text_summary == "tool outcome"
@pytest.mark.asyncio
async def test_flash_saves_and_reads_text_with_verified_receipt_without_device_actions(tmp_path):
import hashlib
@@ -27,6 +27,7 @@ from unittest.mock import Mock
import pytest
from artemis.mcp.action_executor import McpActionExecutor, _ArgError
from artemis.utils.action_target import observation_anchor
def _make_executor(width=1080, height=2400):
@@ -90,6 +91,8 @@ def test_click_index_resolves_to_the_elements_center_and_records_observed_fields
"target_bounds": [37, 480, 963, 560],
"target_resource_id": "android:id/title",
"target_class": "android.widget.TextView",
"target_index": 1,
**observation_anchor(_indexed_state()),
}
assert "target_description" not in recorded
+28 -3
View File
@@ -384,7 +384,10 @@ def test_mobile_manage_task_inject_instruction(temp_trace_env):
ok_res = mobile_manage_task(
action="inject_instruction", trace_id=trace_id, instruction="Scroll down slowly"
)
assert "current model step interrupted" in ok_res["message"]
assert "next safe checkpoint" in ok_res["message"]
assert "not guaranteed" in ok_res["message"]
assert ok_res["guidance"]["id"]
assert ok_res["guidance"]["status"] == "pending"
@pytest.mark.asyncio
@@ -476,11 +479,33 @@ async def test_mobile_inspect_trace_includes_device_serial(temp_trace_env):
"CREATE TABLE traces (trace_id TEXT PRIMARY KEY, step_id TEXT, session_id TEXT, type TEXT, name TEXT, status TEXT, timestamp REAL, duration REAL, payload TEXT)"
)
conn.execute("INSERT INTO sessions VALUES (?, ?, ?)", (trace_id, 1000.0, None))
conn.executemany(
"INSERT INTO steps (step_id,session_id,step_number,timestamp) VALUES (?,?,?,?)",
[(f"actual-{n}", trace_id, n, 1000 + n) for n in range(1, 4)],
)
conn.commit()
conn.close()
res_summary = await mobile_inspect_trace(action="view_summary", trace_id=trace_id)
assert "**Device Serial:** `pixel-target-999`" in res_summary
res_summary = await mobile_inspect_trace(action="view_summary", trace_id=trace_id, limit=2)
assert "**Device Serial:** `pixel-target-999`" in res_summary.content[0].text
assert res_summary.content[0].text.startswith("# ARTEMIS Flash Execution Summary")
assert res_summary.structuredContent["steps"] == {
"items": [
{"step_id": "actual-1", "step_number": 1},
{"step_id": "actual-2", "step_number": 2},
],
"total": 3,
"offset": 0,
"limit": 2,
"next_offset": 2,
}
last_page = await mobile_inspect_trace(
action="view_summary", trace_id=trace_id, offset=2, limit=2
)
assert last_page.structuredContent["steps"]["next_offset"] is None
assert last_page.structuredContent["steps"]["items"] == [
{"step_id": "actual-3", "step_number": 3}
]
def test_trace_store_device_serial(temp_trace_env):
+58
View File
@@ -0,0 +1,58 @@
"""Protocol regressions without starting ARTEMIS or invoking a device."""
import json
from typing import Any
from mcp.server.fastmcp.exceptions import ToolError
from mcp.shared.memory import create_connected_server_and_client_session
import pytest
from apps.admin_console.schemas.task_schema import RunRequest
from artemis.mcp.tool_server import ArtemisFastMCP
@pytest.mark.asyncio
async def test_unknown_arguments_are_rejected_before_dispatch():
mcp = ArtemisFastMCP("strict-input-test", strict_inputs=True)
dispatched = []
@mcp.tool()
def submit(task_desc: str) -> dict:
dispatched.append(task_desc)
return {"ok": True}
with pytest.raises(ToolError, match="未知参数.*goal"):
await mcp.call_tool("submit", {"task_desc": "original", "goal": "silently replace?"})
assert dispatched == []
assert (await mcp.list_tools())[0].inputSchema["additionalProperties"] is False
await mcp.call_tool("submit", {"task_desc": "accepted"})
assert dispatched == ["accepted"]
@pytest.mark.asyncio
async def test_legacy_json_has_structured_content_without_changing_text():
mcp = ArtemisFastMCP("structured-test", strict_inputs=True)
@mcp.tool()
def legacy() -> Any:
return {"items": [{"step_id": "actual-step"}], "offset": 0, "total": 1, "next_offset": None}
content, structured = await mcp.call_tool("legacy", {})
assert json.loads(content[0].text) == structured
assert structured["items"][0]["step_id"] == "actual-step"
async with create_connected_server_and_client_session(mcp) as session:
wire = await session.call_tool("legacy", {})
assert wire.structuredContent == structured
assert json.loads(wire.content[0].text) == structured
rejected = await session.call_tool("legacy", {"goal": "unknown field"})
assert rejected.isError is True
def test_http_submission_rejects_unknown_and_misspelled_fields():
from pydantic import ValidationError
with pytest.raises(ValidationError, match="extra_forbidden"):
RunRequest.model_validate({"goal": "original", "task_desc": "wrong contract"})
assert (
RunRequest.model_validate({"goal": "accepted", "runtime_overrides": {}}).goal == "accepted"
)
@@ -19,6 +19,7 @@ import time
import pytest
from artemis.runtime import cancel_requests as cr
from artemis.runtime.cancellation import CancellationReason
@pytest.fixture(autouse=True)
@@ -91,3 +92,27 @@ async def test_watch_delivers_once_and_clears_marker():
assert await asyncio.wait_for(watcher, timeout=2) is True
assert delivered == [1]
assert not cr.is_cancel_requested(session_id="w-1")
@pytest.mark.asyncio
@pytest.mark.parametrize("cause", [CancellationReason.USER_STOP, CancellationReason.UNKNOWN])
async def test_watch_forwards_only_the_structured_cause(cause):
path = cr.request_cancel(
session_id="with-reason", reason="private stop details", cancel_reason=cause
)[0]
delivered = []
assert await cr.watch_for_cancel_request(
delivered.append, session_id="with-reason", pid=0, pass_reason=True
)
assert delivered == [cause]
assert not path.exists()
def test_legacy_or_unrecognized_marker_reasons_remain_unknown():
path = cr.request_cancel(session_id="legacy", reason="stop from UI")[0]
payload = json.loads(path.read_text(encoding="utf-8"))
for value in [None, "private user instruction", ["user_stop"]]:
payload["cancel_reason"] = value
path.write_text(json.dumps(payload), encoding="utf-8")
assert cr.get_cancel_reason(session_id="legacy") is CancellationReason.UNKNOWN
assert cr.is_cancel_requested(session_id="legacy")
+63
View File
@@ -14,6 +14,47 @@ import pytest
import artemis.runtime.task_control as task_control
def test_transition_timing_keeps_repeated_states_and_separates_pause_and_resource_wait(
tmp_path, monkeypatch
):
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
clock = [10]
monkeypatch.setattr(
task_control,
"time",
SimpleNamespace(**{**vars(task_control.time), "time": lambda: clock[0]}),
)
for at, status in (
(10, "queued"),
(20, "running"),
(25, "running"),
(30, "task_paused"),
(40, "resuming"),
(50, "running"),
):
clock[0] = at
task_control.write_control_state("timed-task", status)
assert task_control.read_state("timed-task")["timing"] == {
"started_at": 10,
"state_started_at": 50,
"seconds": {"queued": 10, "running": 10, "task_paused": 10, "resuming": 10},
}
def test_non_object_control_state_and_invalid_timing_can_be_replaced(tmp_path, monkeypatch):
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
path = task_control.control_dir("broken-state") / "task_control_state.json"
path.parent.mkdir(parents=True)
path.write_text("[]", encoding="utf-8")
assert task_control.read_state("broken-state") == {}
task_control.write_control_state("broken-state", "running")
state = task_control.read_state("broken-state")
state["timing"] = "invalid"
task_control._write_atomic(path, state)
task_control.write_control_state("broken-state", "task_paused")
assert task_control.read_state("broken-state")["timing"]["seconds"] == {}
def _windows_access_error(code: int) -> PermissionError:
error = PermissionError(errno.EACCES, "state file is in use")
error.winerror = code
@@ -423,6 +464,28 @@ def test_guidance_history_distinguishes_submission_from_runner_consumption(tmp_p
assert all(item["status"] == "consumed" and item["consumed_at"] for item in history)
def test_terminal_guidance_reports_no_consumption_and_keeps_original_receipt(tmp_path, monkeypatch):
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
receipt = task_control.add_instruction("task-123", "检查新入口", instruction_id="guidance")
history = task_control.list_instructions("task-123", task_status="cancelled")
assert history[0]["status"] == "unapplied"
assert history[0]["accepted_at"] == receipt["timestamp"]
assert "consumed_at" not in history[0]
assert task_control.list_instructions("task-123")[0]["status"] == "pending"
def test_claimed_guidance_is_visible_and_not_falsely_reported_as_consumed(tmp_path, monkeypatch):
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
task_control.add_instruction("task-123", "检查新入口", instruction_id="guidance")
directory = task_control.control_dir("task-123") / "instructions"
(directory / "guidance.json").rename(directory / "guidance.reading")
assert task_control.list_instructions("task-123")[0]["status"] == "reading"
assert (
task_control.list_instructions("task-123", task_status="completed")[0]["status"]
== "unconfirmed"
)
def test_guidance_history_endpoint_returns_pending_and_consumed(tmp_path, monkeypatch):
from fastapi import FastAPI
from fastapi.testclient import TestClient
+96
View File
@@ -0,0 +1,96 @@
from artemis.runtime.task_progress import progress_snapshot
def test_queued_time_is_not_execution_time_even_with_legacy_started_at():
value = progress_snapshot(job={"status": "queued", "created_at": 10, "started_at": 10}, now=50)
assert value["phase"] == "queued"
assert value["durations"]["queued"] == 40
assert value["durations"]["execution"] is None
def test_progress_splits_wait_pause_review_and_export_without_double_counting():
value = progress_snapshot(
job={
"status": "completed",
"created_at": 10,
"worker_started_at": 20,
"ended_at": 120,
"finalized_at": 125,
},
control={
"status": "running",
"timing": {
"started_at": 20,
"state_started_at": 80,
"seconds": {"queued": 10, "task_paused": 20, "resuming": 10, "running": 20},
},
},
reviews=[{"started_at": 90, "ended_at": 110}, {"started_at": 100, "ended_at": 115}],
now=500,
)
assert value["phase"] == "finished"
assert value["durations"] == {
"total": 115,
"queued": 10,
"resource_wait": 20,
"paused": 20,
"final_review": 25,
"execution": 35,
"report_export": 5,
}
def test_active_review_has_its_own_phase_and_terminal_review_is_not_still_active():
review = {"started_at": 30, "status": "running", "updated_at": 35}
live = progress_snapshot(job={"status": "running"}, reviews=[review], now=40)
assert live["phase"] == "final_review"
assert live["durations"]["final_review"] == 10
stopped = progress_snapshot(
job={"status": "cancelled", "ended_at": 45}, reviews=[review], now=500
)
assert stopped["phase"] == "finished"
assert stopped["durations"]["final_review"] == 5
assert not stopped["review_timing_complete"]
def test_legacy_missing_intervals_are_unknown_rather_than_zero_or_model_speed():
value = progress_snapshot(
session={"status": "completed", "start_time": 20, "end_time": 100}, now=500
)
assert value["durations"]["queued"] is None
assert value["durations"]["execution"] is None
assert value["durations"]["paused"] is None
assert value["durations"]["total"] is None
assert value["observed_seconds"] == 80
def test_waiting_for_resource_and_pause_are_not_running():
for status, phase in (
("queued", "waiting_resources"),
("resuming", "waiting_resources"),
("task_paused", "paused"),
):
value = progress_snapshot(
job={"status": "running", "worker_started_at": 10}, control={"status": status}, now=50
)
assert value["phase"] == phase
def test_terminal_without_end_timestamp_does_not_keep_increasing_or_guess_duration():
job = {"status": "completed", "created_at": 10, "worker_started_at": 20}
first = progress_snapshot(job=job, now=100)
assert first == progress_snapshot(job=job, now=500)
assert first["observed_at"] is None
assert all(first["durations"][key] is None for key in ("total", "execution", "report_export"))
def test_export_in_progress_keeps_total_current_without_extending_execution():
value = progress_snapshot(
job={"status": "finalizing", "created_at": 10, "worker_started_at": 100, "ended_at": 200},
now=230,
)
assert value["phase"] == "report_export"
assert value["observed_at"] == 230
assert value["durations"]["total"] == 220
assert value["durations"]["report_export"] == 30
assert value["worker_ended_at"] == 200
+8 -3
View File
@@ -19,11 +19,13 @@ import asyncio
import pytest
from artemis.runtime import cancel_requests
from artemis.runtime.cancellation import CancellationReason
from artemis.sdk.agent import Agent
@pytest.mark.asyncio
async def test_external_cancel_marker_cancels_running_task(tmp_path, monkeypatch):
@pytest.mark.parametrize("cause", [CancellationReason.USER_STOP, CancellationReason.UNKNOWN])
async def test_external_cancel_marker_cancels_running_task(tmp_path, monkeypatch, cause):
monkeypatch.setattr(cancel_requests, "get_temp_dir", lambda _subfolder=None: tmp_path / "c")
monkeypatch.setenv("ARTEMIS_CANCEL_POLL_SECONDS", "0.05")
@@ -38,9 +40,12 @@ async def test_external_cancel_marker_cancels_running_task(tmp_path, monkeypatch
await asyncio.sleep(0.15)
assert not agent._current_task.done()
cancel_requests.request_cancel(session_id="sess-cancel", reason="stop from UI")
cancel_requests.request_cancel(
session_id="sess-cancel", reason="stop from UI", cancel_reason=cause
)
with pytest.raises(asyncio.CancelledError):
with pytest.raises(asyncio.CancelledError) as error:
await asyncio.wait_for(agent._current_task, timeout=2)
assert error.value.args == (cause,)
await asyncio.wait_for(watcher, timeout=2)
assert not cancel_requests.is_cancel_requested(session_id="sess-cancel")
+225
View File
@@ -0,0 +1,225 @@
"""Cancellation provenance must survive the real runtime-to-model path."""
import asyncio
from copy import deepcopy
import pytest
from artemis.llm.final_review import FinalReviewBudget
from artemis.llm.reliability import LLMExhaustedError
from artemis.runtime.cancellation import CancellationReason
from artemis.runtime import task_control
from artemis.sdk.agent import Agent
from artemis.services import llm
class Engine:
current_session_id = "cancellation-test"
current_step_id = None
def __init__(self):
self.records = []
def record_trace(self, **record):
self.records.append(deepcopy(record))
def _publish(self, *_):
pass
@property
def attempt(self):
return next(r["payload"] for r in reversed(self.records) if r["type"] == "llm_diagnostic")
@pytest.mark.asyncio
async def test_guidance_cancellation_reaches_request_without_retry(tmp_path, monkeypatch):
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
engine = Engine()
started = asyncio.Event()
calls = 0
async def model():
nonlocal calls
calls += 1
started.set()
await asyncio.Event().wait()
control = task_control.TaskControl("task", object())
review = FinalReviewBudget(
engine, attempt_id="final#1", request_timeout=10, max_retries=5, retry_delay=0
)
running = asyncio.create_task(
control.run_interruptibly(
review.run(lambda: llm._run_with_recovery(model, max_retries=5, engine=engine))
)
)
await asyncio.wait_for(started.wait(), 2)
task_control.add_instruction("task", "Use the newly observed page")
assert await asyncio.wait_for(running, 2) == (True, None)
assert engine.attempt["phase"] == "cancelled"
assert engine.attempt["cancel_reason"] == "guidance_replaced"
assert review.data["status"] == "cancelled"
assert review.data["cancel_reason"] == "guidance_replaced"
assert calls == 1
@pytest.mark.asyncio
async def test_sdk_stop_reaches_request_without_retry():
engine = Engine()
started = asyncio.Event()
async def model():
started.set()
await asyncio.Event().wait()
agent = Agent.__new__(Agent)
agent._current_task = asyncio.create_task(
llm._run_with_recovery(model, max_retries=5, engine=engine)
)
await asyncio.wait_for(started.wait(), 2)
agent.stop_current_task(reason=CancellationReason.USER_STOP)
with pytest.raises(asyncio.CancelledError):
await agent._current_task
assert engine.attempt["phase"] == "cancelled"
assert engine.attempt["cancel_reason"] == "user_stop"
@pytest.mark.asyncio
async def test_outer_timeout_is_not_a_request_timeout():
engine = Engine()
async def model():
await asyncio.Event().wait()
with pytest.raises(TimeoutError):
await llm.invoke_llm_with_timeout_message(
llm._run_with_recovery(model, max_retries=5, engine=engine),
timeout_seconds=0,
hard_timeout=0,
)
assert engine.attempt["phase"] == "cancelled"
assert engine.attempt["cancel_reason"] == "outer_timeout"
assert engine.attempt["timeout_scope"] == "outer"
@pytest.mark.asyncio
async def test_sdk_stop_preserves_cause_through_nested_request_wrappers(tmp_path, monkeypatch):
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
engine = Engine()
started = asyncio.Event()
async def model():
started.set()
await asyncio.Event().wait()
control = task_control.TaskControl("task", object())
agent = Agent.__new__(Agent)
agent._current_task = asyncio.create_task(
control.run_interruptibly(
llm.invoke_llm_with_timeout_message(
llm._run_with_recovery(model, max_retries=5, engine=engine), timeout_seconds=60
)
)
)
await asyncio.wait_for(started.wait(), 2)
agent.stop_current_task()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(agent._current_task, 2)
assert engine.attempt["phase"] == "cancelled"
assert engine.attempt["cancel_reason"] == "user_stop"
@pytest.mark.asyncio
async def test_request_timer_is_identified_without_triggering_an_extra_request():
engine = Engine()
token = llm._FALLBACK_AVAILABLE.set(True)
calls = 0
async def model():
nonlocal calls
calls += 1
await asyncio.Event().wait()
try:
with pytest.raises(LLMExhaustedError):
await llm._run_with_recovery(model, request_timeout=0.01, max_retries=0, engine=engine)
finally:
llm._FALLBACK_AVAILABLE.reset(token)
assert engine.attempt["phase"] == "failed"
assert engine.attempt["timeout_scope"] == "request"
assert "cancel_reason" not in engine.attempt
assert calls == 1
@pytest.mark.asyncio
async def test_unattributed_cancellation_does_not_infer_a_user_action_or_store_its_text():
engine = Engine()
started = asyncio.Event()
async def model():
started.set()
await asyncio.Event().wait()
running = asyncio.create_task(llm._run_with_recovery(model, max_retries=5, engine=engine))
await asyncio.wait_for(started.wait(), 2)
running.cancel("private user guidance")
with pytest.raises(asyncio.CancelledError):
await running
assert engine.attempt["cancel_reason"] == "unknown"
assert "private user guidance" not in repr(engine.records)
@pytest.mark.asyncio
async def test_http_499_preserves_server_id_but_does_not_infer_local_cancellation():
engine = Engine()
token = llm._FALLBACK_AVAILABLE.set(True)
class ServerError(Exception):
status_code = 499
request_id = "server-request-499"
async def model():
raise ServerError("Request closed")
try:
with pytest.raises(LLMExhaustedError):
await llm._run_with_recovery(model, max_retries=0, engine=engine)
finally:
llm._FALLBACK_AVAILABLE.reset(token)
assert engine.attempt["phase"] == "failed"
assert engine.attempt["http_status"] == 499
assert engine.attempt["server_request_id"] == "server-request-499"
assert "cancel_reason" not in engine.attempt
@pytest.mark.asyncio
async def test_pause_cancellation_uses_the_existing_checkpoint(tmp_path, monkeypatch):
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
engine = Engine()
started = asyncio.Event()
paused = asyncio.Event()
events = []
class Lock:
def release(self):
events.append("released")
async def model():
started.set()
await asyncio.Event().wait()
async def checkpoint(**_kwargs):
events.append("checkpoint")
paused.set()
control = task_control.TaskControl("task", Lock())
monkeypatch.setattr(control, "checkpoint", checkpoint)
running = asyncio.create_task(
control.run_interruptibly(llm._run_with_recovery(model, max_retries=5, engine=engine))
)
await asyncio.wait_for(started.wait(), 2)
task_control.request_pause("task")
await asyncio.wait_for(paused.wait(), 2)
assert await running == (True, None)
assert engine.attempt["cancel_reason"] == "task_paused"
assert events == ["checkpoint"]
+2
View File
@@ -191,6 +191,8 @@ async def test_total_budget_expiry_is_not_user_cancellation_or_a_new_retry(
stored = engine.latest("llm_diagnostic")
assert stored["phase"] == "failed"
assert stored["category"] == "final_review_budget_exhausted"
assert stored["timeout_scope"] == "final_review"
assert "cancel_reason" not in stored
@pytest.mark.asyncio
+4 -1
View File
@@ -77,7 +77,10 @@ def test_minimal_list_shows_hint_for_empty_inputs_and_errors():
)
text, items, labels = format_minimal_list_with_elements(elements[1:], 1000, 1000)
lines = text.splitlines()
assert lines[0] == "[1] Hint: 'Search settings' | Bounds: [0,0][100,100]"
assert (
lines[0]
== "[1] Hint: 'Search settings' | Bounds: [0,0][100,100] | Editable: true | Value: empty"
)
assert lines[1] == "[2] Text: 'abc' | Bounds: [0,200][100,300] | Error: 'Password too short'"
assert len(lines) == 2 and labels == ["1", "2"]
assert items[0]["is_hint"] is True and items[0]["text"] == "Search settings"
+93
View File
@@ -0,0 +1,93 @@
"""A hint, OCR label and real editable value are different kinds of proof."""
from types import SimpleNamespace
import pytest
from artemis.drivers.android.hierarchy import parse_ui_hierarchy
from artemis.utils.action_target import observation_anchor
from artemis.utils.input_state import input_state, verify_input
from artemis.utils.ui_filter import filter_ui_hierarchy
from artemis.utils.visualization import format_minimal_list_with_elements
@pytest.mark.parametrize(
("extra", "status"),
[
({"text": "example", "hint": "example", "showing-hint-text": "true"}, "empty"),
({"text": "example", "hint": "example"}, "unknown"),
({"text": "example", "hint": "example", "showing-hint-text": "false"}, "present"),
({"text": "", "hint": "example"}, "empty"),
({"hint": "example"}, "unknown"),
({"text": "actual"}, "present"),
],
)
def test_editable_value_evidence(extra, status):
evidence = input_state({"class": "android.widget.EditText", **extra})
assert evidence["input_value_state"] == status
if status != "present":
assert evidence["input_value"] in (None, "")
def test_static_json_and_ocr_cannot_prove_input():
static = {"class": "android.widget.TextView", "text": '{"pairing":"example"}'}
assert input_state(static) == {}
node = {
"class": "android.widget.EditText",
"text": "",
"hint": "example",
"bounds": "[0,0][100,100]",
"ocr_elements": [{"text": "example", "bounds": [0, 0, 100, 100]}],
}
lines, indexed, _ = format_minimal_list_with_elements([node], 100, 100)
assert len(indexed) == 1
assert indexed[0]["input_value_state"] == "empty"
assert "Hint:" in lines
def test_filter_keeps_empty_input_and_does_not_promote_child_hint():
node = {
"class": "android.widget.EditText",
"editable": True,
"text": "",
"bounds": "[0,0][100,100]",
"children": [
{"class": "android.widget.TextView", "text": "example", "bounds": "[0,0][100,100]"}
],
}
filtered = filter_ui_hierarchy([node])
assert filtered[0]["text"] == ""
assert input_state(filtered[0])["input_value_state"] == "empty"
def test_fresh_value_must_belong_to_the_actual_input():
node = {
"class": "android.widget.EditText",
"resource-id": "pair",
"text": "actual",
"focused": True,
}
assert verify_input([node], {"target_resource_id": "pair"}) == "present"
assert verify_input([node], {"target_resource_id": "other"}) == "unknown"
assert verify_input([{**node, "password": True}]) == "unknown"
def test_driver_parser_keeps_showing_hint_and_empty_editable_nodes():
nodes = parse_ui_hierarchy(
'<hierarchy><node class="android.widget.EditText" bounds="[0,0][100,100]" editable="true" text="example" hint="example" showing-hint-text="true" focused="true"/></hierarchy>'
)
assert len(nodes) == 1
assert nodes[0]["input_value"] == ""
assert nodes[0]["input_value_state"] == "empty"
assert nodes[0]["focused"] is True
def test_past_anchor_does_not_change_when_index_is_reused():
state = SimpleNamespace(
latest_screenshot="before.png", indexed_elements=[{"text": "Connect", "center": [1, 2]}]
)
old = observation_anchor(state)
state.latest_screenshot = "after.png"
state.indexed_elements = [{"text": "Keyboard key", "center": [3, 4]}]
assert old["target_image_name"] == "before"
assert observation_anchor(state)["target_snapshot_id"] != old["target_snapshot_id"]
+276
View File
@@ -0,0 +1,276 @@
"""Reject unfinished Responses output before parsers or tool consumers see it."""
import asyncio
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock
import httpx
from langchain_core.messages import AIMessage, AIMessageChunk, HumanMessage, ToolMessage
from langchain_core.runnables import RunnableLambda
from pydantic import BaseModel, SecretStr
import pytest
import pytest_asyncio
from artemis.llm.reliability import FailureCategory, LLMExhaustedError
from artemis.llm.response_guard import ResponseIntegrityError, ensure_response_complete
from artemis.llm.router import ModelEndpoint, ModelFactory
from artemis.services import llm as llm_service
from artemis.services.llm import RobustChatModelWrapper, acomplete
def response_body(status, arguments, name):
return {
"id": "resp_test",
"object": "response",
"created_at": 1,
"status": status,
"model": "test-model",
"output": [
{
"type": "function_call",
"id": "fc_test",
"call_id": "call_test",
"name": name,
"arguments": arguments,
"status": "completed",
}
],
"incomplete_details": {"reason": "max_output_tokens"} if status == "incomplete" else None,
"usage": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13},
}
def response_stream(status, arguments, name):
body = response_body(status or "in_progress", arguments, name)
events = [
{"type": "response.created", "response": {**body, "status": "in_progress", "output": []}},
{
"type": "response.output_item.added",
"output_index": 0,
"item": {**body["output"][0], "arguments": "", "status": "in_progress"},
},
{
"type": "response.function_call_arguments.delta",
"output_index": 0,
"item_id": "fc_test",
"delta": arguments,
},
]
if status:
events.append({"type": "response." + status, "response": body})
return "".join("data: " + json.dumps(event) + "\n\n" for event in events) + "data: [DONE]\n\n"
@pytest_asyncio.fixture
async def transport_case(monkeypatch, tmp_path):
from langchain_openai import ChatOpenAI
import openai
requests, outputs, clients, events = [], [], [], []
def handle(request):
payload = json.loads(request.content)
requests.append(payload)
status, arguments = outputs.pop(0)
name = payload.get("tools", [{"name": "finish"}])[0]["name"]
if payload.get("stream"):
return httpx.Response(
200,
text=response_stream(status, arguments, name),
headers={"content-type": "text/event-stream"},
)
return httpx.Response(200, json=response_body(status, arguments, name))
def http_client(**kwargs):
client = httpx.AsyncClient(transport=httpx.MockTransport(handle), **kwargs)
clients.append(client)
return client
monkeypatch.setattr(openai, "DefaultAsyncHttpxClient", http_client)
monkeypatch.setattr(llm_service.settings, "OPENAI_API_KEY", SecretStr("test-secret"))
monkeypatch.setattr(llm_service.settings, "OPENAI_BASE_URL", "https://responses.example/v1")
monkeypatch.setattr(llm_service.settings, "OPENAI_MAX_RETRIES", 5)
monkeypatch.setattr(llm_service.settings, "OPENAI_RETRY_INTERVAL_SECONDS", 0)
monkeypatch.delenv("ARTEMIS_FAKE_LLM", raising=False)
monkeypatch.setattr(llm_service, "PAUSE_FILE", tmp_path / ".artemis_paused")
monkeypatch.setattr(llm_service, "_wait_for_resume", AsyncMock(return_value=False))
monkeypatch.setattr(llm_service, "_get_current_data_engine", lambda: None)
monkeypatch.setattr(
llm_service, "_record_llm_event", lambda *args, **kwargs: events.append(args)
)
llm_service._ENDPOINT_BREAKER._states.clear()
llm_service._NON_STREAMING_ENDPOINTS.clear()
endpoint = ModelEndpoint(model_name="test-model", max_retries=0)
model = ModelFactory.create_model(endpoint)
assert isinstance(model, ChatOpenAI)
yield SimpleNamespace(
model=model, endpoint=endpoint, requests=requests, outputs=outputs, events=events
)
for client in clients:
await client.aclose()
llm_service._ENDPOINT_BREAKER._states.clear()
llm_service._NON_STREAMING_ENDPOINTS.clear()
def wrapped(case):
return RobustChatModelWrapper(case.model, endpoint=case.endpoint).bind_tools(
[
{
"type": "function",
"function": {
"name": "finish",
"description": "Record the verdict",
"parameters": {"type": "object", "properties": {"passed": {"type": "boolean"}}},
},
}
]
)
@pytest.mark.asyncio
@pytest.mark.parametrize("status", [None, "incomplete", "failed"])
@pytest.mark.parametrize("arguments", ['{"passed":true}', '{"passed":true'])
async def test_unfinished_stream_never_returns_a_verdict_or_tool_call(
transport_case, status, arguments
):
case = transport_case
case.outputs.append((status, arguments))
with pytest.raises(LLMExhaustedError) as failure:
await wrapped(case).complete([HumanMessage(content="Verify the task")])
assert failure.value.failure.category is FailureCategory.INVALID_OUTPUT
assert len(case.requests) == 1
resets = [payload for event, payload in case.events if event == "llm_stream_reset"]
assert len(resets) == 1 and resets[0]["action"] == "discard"
assert not llm_service._NON_STREAMING_ENDPOINTS
@pytest.mark.asyncio
@pytest.mark.parametrize("arguments", ['{"passed":true', ""])
async def test_completed_event_cannot_repair_invalid_tool_json(transport_case, arguments):
case = transport_case
case.outputs.append(("completed", arguments))
with pytest.raises(LLMExhaustedError) as failure:
await wrapped(case).complete([])
assert failure.value.failure.category is FailureCategory.INVALID_OUTPUT
assert len(case.requests) == 1
@pytest.mark.parametrize("arguments", [None, "[]", "null", '{"passed":true'])
def test_original_function_call_arguments_must_be_a_complete_json_object(arguments):
message = AIMessage(
content=[{"type": "function_call", "arguments": arguments}],
response_metadata={"model_provider": "openai", "status": "completed"},
)
with pytest.raises(ResponseIntegrityError, match="invalid_tool_arguments"):
ensure_response_complete(message)
@pytest.mark.asyncio
async def test_completed_stream_returns_original_valid_arguments(transport_case):
case = transport_case
case.outputs.append(("completed", '{"passed":true}'))
result = await wrapped(case).complete([])
assert result.response_metadata["status"] == "completed"
assert result.tool_calls[0]["args"] == {"passed": True}
@pytest.mark.asyncio
@pytest.mark.parametrize("recovers", [False, True])
async def test_integrity_retry_is_bounded_and_keeps_existing_tool_evidence(
transport_case, recovers
):
case = transport_case
case.endpoint.max_retries = 5
case.outputs.extend(
[(None, '{"passed":false}'), ("completed" if recovers else None, '{"passed":true}')]
)
messages = [
HumanMessage(content="Inspect then verify"),
AIMessage(content="", tool_calls=[{"name": "inspect", "args": {}, "id": "prior_call"}]),
ToolMessage(content="Existing screenshot evidence", tool_call_id="prior_call"),
]
if recovers:
result = await wrapped(case).complete(messages)
assert result.tool_calls[0]["args"] == {"passed": True}
else:
with pytest.raises(LLMExhaustedError):
await wrapped(case).complete(messages)
assert len(case.requests) == 2
assert case.requests[0]["input"] == case.requests[1]["input"]
assert any(
item.get("call_id") == "prior_call" and item.get("output") == "Existing screenshot evidence"
for item in case.requests[1]["input"]
)
assert isinstance(messages[-1], ToolMessage)
@pytest.mark.asyncio
@pytest.mark.parametrize("status", ["incomplete", "in_progress", "queued", "cancelled"])
async def test_non_streaming_response_also_requires_completed(transport_case, status):
case = transport_case
case.outputs.append((status, '{"passed":true}'))
with pytest.raises(LLMExhaustedError) as failure:
await wrapped(case).ainvoke([])
assert failure.value.failure.category is FailureCategory.INVALID_OUTPUT
@pytest.mark.asyncio
@pytest.mark.parametrize("include_raw", [False, True])
async def test_structured_output_is_checked_before_returning_parsed_verdict(
transport_case, include_raw
):
class Verdict(BaseModel):
passed: bool
case = transport_case
case.outputs.append(("incomplete", '{"passed":true}'))
model = RobustChatModelWrapper(case.model, endpoint=case.endpoint)
with pytest.raises(LLMExhaustedError):
await model.with_structured_output(Verdict, include_raw=include_raw).ainvoke("Verify")
@pytest.mark.asyncio
async def test_downstream_parser_never_receives_incomplete_response(transport_case):
case = transport_case
case.outputs.append(("incomplete", '{"passed":true}'))
parsed = []
chain = wrapped(case) | RunnableLambda(lambda value: parsed.append(value))
with pytest.raises(LLMExhaustedError):
await chain.ainvoke([])
assert parsed == []
@pytest.mark.asyncio
async def test_raw_model_compatibility_path_does_not_accept_missing_terminal_event(transport_case):
case = transport_case
case.outputs.append((None, '{"passed":true}'))
with pytest.raises(ValueError):
await acomplete(case.model, [])
assert len(case.requests) == 1
@pytest.mark.asyncio
async def test_cancellation_still_closes_stream_without_retry(transport_case):
started, closed = asyncio.Event(), []
class StreamingModel:
async def astream(self, *args, **kwargs):
try:
yield AIMessageChunk(
content="partial", response_metadata={"model_provider": "openai"}
)
started.set()
await asyncio.Event().wait()
finally:
closed.append(True)
task = asyncio.create_task(RobustChatModelWrapper(StreamingModel()).complete([]))
await asyncio.wait_for(started.wait(), timeout=2)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert closed == [True]
assert not transport_case.requests
resets = [payload for event, payload in transport_case.events if event == "llm_stream_reset"]
assert len(resets) == 1 and resets[0]["reason"] == "incomplete_response"
+57
View File
@@ -271,6 +271,63 @@ async def test_receipt_disk_failure_preserves_successful_write_result(monkeypatc
assert outcome.side_effect != "none"
@pytest.mark.asyncio
async def test_write_popup_source_updates_cached_observation_target():
runtime = PluginRuntime.__new__(PluginRuntime)
runtime.ctx = None
runtime.bindings = {
"write": ("visual-test", Tool(name="write", inputSchema={"type": "object"}))
}
runtime.packages = {
"visual-test": (
manifest(
capabilities={
"kind": "visual",
"tools": {
"observe": {
"effect": "read",
"observation": True,
"target_argument": "window_id",
},
"write": {"effect": "write", "target_argument": "window_id"},
},
}
),
SimpleNamespace(argument_constraints={}, secrets={}),
None,
)
}
runtime.sessions = {
"visual-test": SimpleNamespace(
call=AsyncMock(
return_value=CallToolResult(
content=[ImageContent(type="image", mimeType="image/png", data="aW1hZ2U=")],
structuredContent={
"artemis": {
"version": 1,
"execution": "completed",
"source": {"kind": "computer", "target_id": "123"},
}
},
)
),
stop_uncertain=False,
)
}
runtime.ensure_resources_ready = lambda: None
runtime._image_targets = {}
runtime.last_images = {}
runtime._observation_calls = {"visual-test": ("observe", {"pid": 7, "window_id": 10})}
runtime._recordings = None
first = await runtime.call("write", {"pid": 7, "window_id": 10})
assert first.status == "success"
assert first.pre_image_valid is False
assert runtime._image_targets["visual-test"][0] == "123"
assert runtime._observation_calls["visual-test"] == ("observe", {"pid": 7, "window_id": 123})
second = await runtime.call("write", {"pid": 7, "window_id": 123})
assert second.pre_image_valid is True
@pytest.mark.asyncio
async def test_v2_observation_without_task_context_never_starts_recording(monkeypatch):
runtime = PluginRuntime.__new__(PluginRuntime)
+75 -1
View File
@@ -18,6 +18,7 @@ from artemis.llm.diagnostics import (
)
from artemis.llm.router import ModelEndpoint
from artemis.services import llm
from artemis.runtime.cancellation import CancellationReason
class Engine:
@@ -121,6 +122,10 @@ async def test_headers_without_body_timeout_then_retry_keeps_evidence(monkeypatc
assert failed["usage"] is None and "first_output_at" not in failed
assert final["request_id"] == failed["request_id"]
assert final["attempt_id"] != failed["attempt_id"] and final["attempt"] == 2
assert final["previous_attempt_id"] == failed["attempt_id"]
retry = next(r["payload"] for r in engine.records if r["name"] == "llm_retry")
assert retry["request_id"] == failed["request_id"]
assert retry["attempt_id"] == failed["attempt_id"]
assert final["phase"] == "completed" and final["usage"]["total_tokens"] == 12
assert (
final["started_at"] <= final["headers_at"] <= final["first_output_at"] <= final["ended_at"]
@@ -226,6 +231,72 @@ async def test_concurrent_requests_do_not_mix_server_ids():
assert second["server_request_id"] == "two"
@pytest.mark.asyncio
async def test_diagnostic_ids_survive_streaming_and_structured_completion_without_header_bodies():
engine = Engine()
attempt = RequestAttempt(
engine, request_id="local", attempt=1, timeout=300, max_retries=1, metadata={}
)
token = ACTIVE_ATTEMPT.set(attempt)
traceparent = "00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-01"
try:
request = httpx.Request(
"POST", "https://example.test/responses", headers={"x-client-request-id": "caller-id"}
)
await observe_request_headers(request)
await observe_response_headers(
httpx.Response(
200,
headers={
"x-request-id": "server-id",
"x-trace-id": "gateway-id",
"traceparent": traceparent,
"set-cookie": "private cookie",
"authorization": "private credential",
},
)
)
attempt.chunk(AIMessageChunk(content="hello", response_metadata={"id": "resp-first"}))
attempt.finish(
"completed",
response={
"raw": AIMessageChunk(
content="hello", response_metadata={"id": "resp-final", "status": "completed"}
),
"parsed": {"private": "business details"},
},
)
finally:
ACTIVE_ATTEMPT.reset(token)
assert request.headers["x-client-request-id"] == "caller-id"
assert attempt.data["client_request_id"] == "caller-id"
assert attempt.data["server_request_id"] == "server-id"
assert attempt.data["server_trace_id"] == "gateway-id"
assert attempt.data["traceparent"] == traceparent
assert attempt.data["response_id"] == "resp-final"
assert attempt.data["response_status"] == "completed"
assert "private" not in json.dumps(engine.records, default=str)
def test_invalid_or_overlong_correlation_headers_are_omitted():
attempt = RequestAttempt(
Engine(), request_id="local", attempt=1, timeout=300, max_retries=1, metadata={}
)
attempt.headers(
httpx.Response(
200,
headers={
"x-request-id": "x" * 257,
"x-trace-id": "not an id",
"traceparent": "not a trace",
},
)
)
assert attempt.data["server_request_id"] is None
assert attempt.data["server_trace_id"] is None
assert "traceparent" not in attempt.data
def test_error_redaction_and_image_headers():
import base64
import io
@@ -372,11 +443,14 @@ async def test_cancelling_retry_wait_does_not_leave_a_live_request(monkeypatch):
task = asyncio.create_task(llm._run_with_recovery(timeout, max_retries=1))
await asyncio.wait_for(waiting.wait(), 2)
task.cancel()
task.cancel(CancellationReason.GUIDANCE_REPLACED)
with pytest.raises(asyncio.CancelledError):
await task
assert engine.records[-1]["payload"]["phase"] == "cancelled"
assert "模型请求" in engine.records[-1]["payload"]["error"]
assert engine.records[-1]["payload"]["cancel_reason"] == "guidance_replaced"
assert engine.records[-1]["payload"]["cancel_phase"] == "retry_wait"
assert "timeout_scope" not in engine.records[-1]["payload"]
def test_diagnostic_api_limits_preview_but_downloads_complete_history(tmp_path, monkeypatch):
+66 -4
View File
@@ -361,7 +361,7 @@ async def test_launch_app_failure_when_top_task_is_another_app(
assert success is False
assert f"Failed to launch {SETTINGS}" in error_msg
assert mock_controller.launch_app.call_count == 2
assert mock_controller.terminate_app.call_count == 1
mock_controller.terminate_app.assert_not_called()
@pytest.mark.asyncio
@@ -399,12 +399,13 @@ async def test_launch_timeout_message_names_foreground_package_and_task(
@patch("artemis.utils.app_launch_utils.get_foreground_task")
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
async def test_launch_app_null_focus_keeps_polling_without_reading_task(
async def test_launch_app_null_focus_without_task_evidence_keeps_polling(
mock_get_foreground, mock_controller_cls, mock_get_task, mock_context
):
"""mCurrentFocus=null is a loading state: no task lookup, then success once focused."""
"""Unknown focus and task state remain loading until a focused window appears."""
_controller(mock_controller_cls)
mock_get_foreground.side_effect = [None, None, YOUTUBE]
mock_get_task.return_value = None
success, error_msg = await launch_app_with_retries(
mock_context, YOUTUBE, max_retries=1, max_poll_seconds=3
@@ -412,7 +413,67 @@ async def test_launch_app_null_focus_keeps_polling_without_reading_task(
assert success is True
assert error_msg is None
mock_get_task.assert_not_called()
assert mock_get_task.call_count == 2
@pytest.mark.asyncio
@pytest.mark.parametrize(
"dump, ready",
[
(DUMP_SETTINGS_TASK_WITH_SEARCH_ON_TOP, True),
(DUMP_NO_RESUMED, False),
(DUMP_HOME, False),
("", False),
],
ids=["resumed-app", "visible-only", "other-app", "unknown"],
)
async def test_null_focus_requires_resumed_foreground_task_evidence(mock_context, dump, ready):
with (
patch("artemis.utils.app_launch_utils.UnifiedMobileController") as controller_cls,
patch(
"artemis.utils.app_launch_utils.get_current_foreground_package_async",
new_callable=AsyncMock,
return_value=None,
),
patch(
"artemis.utils.app_launch_utils.get_foreground_task",
return_value=parse_foreground_task(dump),
),
patch("artemis.utils.app_launch_utils.asyncio.sleep", new_callable=AsyncMock),
):
controller = _controller(controller_cls)
success, detail = await launch_app_with_retries(
mock_context, SETTINGS, max_retries=2, max_poll_seconds=1
)
assert success is ready
if ready:
assert detail is None
controller.launch_app.assert_awaited_once_with(SETTINGS)
else:
assert "Current foreground: None" in detail
assert controller.launch_app.await_count == 2
controller.terminate_app.assert_not_called()
@pytest.mark.asyncio
async def test_initial_launch_skips_relaunch_with_null_focus_and_resumed_app_task(mock_context):
with (
patch(
"artemis.utils.app_launch_utils.get_current_foreground_package_async",
new_callable=AsyncMock,
return_value=None,
),
patch(
"artemis.utils.app_launch_utils.get_foreground_task",
return_value=parse_foreground_task(DUMP_SETTINGS_TASK_WITH_SEARCH_ON_TOP),
),
patch(
"artemis.utils.app_launch_utils.launch_app_with_retries", new_callable=AsyncMock
) as launch,
):
result = await _handle_initial_app_launch(mock_context, SETTINGS)
assert result.locked_app_initial_launch_success is True
launch.assert_not_awaited()
# ---------------------------------------------------------------------------
@@ -467,6 +528,7 @@ async def test_transient_launch_failure_is_bounded_and_keeps_cause(mock_context)
assert result is False
assert detail == "device offline"
assert controller.launch_app.await_count == 3
controller.terminate_app.assert_not_called()
poll.assert_not_called()