feat: prepare 2.2.0 with UI and execution fixes
Add dark theme, richer Markdown and readable plugin results. Improve Android input evidence, response integrity and diagnostics. Unify lifecycle, guidance receipts and evidence sources across clients. Keep direct action dispatch without an added AI quota approval layer. Align version metadata at 2.2.0 and bundle Helper 1.2.2. Validation: 3063 backend tests before the version bump; 44 after. Frontend: 317 tests and production build passed.
This commit is contained in:
1 parent
83bcfd54d5
commit
fac4ccb760
146 files changed
+5689
-2033
No files matched your search
@@ -1173,6 +1173,112 @@ def test_android_artifact_export_does_not_follow_other_session_links(isolated):
|
||||
assert reports.session_artifacts({"id": "session"}) == []
|
||||
|
||||
|
||||
def test_report_source_inventory_preserves_human_verdict_and_raw_evidence(
|
||||
isolated, tmp_path, monkeypatch
|
||||
):
|
||||
from apps.admin_console.services import evidence_service, task_inspection
|
||||
|
||||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||||
sid = job["session_id"]
|
||||
action = {
|
||||
"action": "plug_read",
|
||||
"plugin_id": "data-reader",
|
||||
"call_id": "recorded-call",
|
||||
"read_only": True,
|
||||
"source": {"kind": "api", "target_id": "business-service", "label": "安卓业务后台"},
|
||||
}
|
||||
with sqlite3.connect(isolated.path) as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO sessions(session_id,initial_goal,start_time,end_time,status) VALUES (?,?,?,?,?)",
|
||||
(sid, "检查业务状态", 1700000010, 1700000120, "completed"),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO steps(step_id,session_id,step_number,action_taken) VALUES (?,?,?,?)",
|
||||
("actual-step", sid, 7, json.dumps(action)),
|
||||
)
|
||||
isolated.save_case_review(sid, "failed", "用户指出业务状态仍未落库")
|
||||
monkeypatch.setattr(evidence_service, "TRACES_PATH", reports.TRACES_PATH)
|
||||
report = reports.session_report(sid)
|
||||
source = report["evidence_sources"]["sources"][0]
|
||||
assert source["category"] == "auxiliary"
|
||||
assert source["references"][0]["step_id"] == "actual-step"
|
||||
assert report["test_result"]["status"] == "failed"
|
||||
assert report["test_result"]["source"] == "human"
|
||||
assert report["steps"][0]["action_taken"] == action
|
||||
index = evidence_service.export_index(sid, report, [])
|
||||
assert index["evidence_sources"] == report["evidence_sources"]
|
||||
monkeypatch.setattr(task_inspection, "store", isolated)
|
||||
monkeypatch.setattr(task_inspection, "session_repo", reports.SessionRepository())
|
||||
assert (
|
||||
task_inspection.inspect_task(sid, "report")["evidence_sources"]
|
||||
== report["evidence_sources"]
|
||||
)
|
||||
from docx import Document
|
||||
from openpyxl import load_workbook
|
||||
|
||||
word = tmp_path / "sources.docx"
|
||||
sheet = tmp_path / "sources.xlsx"
|
||||
reports.word_report(report, word)
|
||||
reports.excel_report(report, sheet)
|
||||
document = Document(word)
|
||||
word_text = " ".join(
|
||||
[p.text for p in document.paragraphs]
|
||||
+ [cell.text for table in document.tables for row in table.rows for cell in row.cells]
|
||||
)
|
||||
workbook = load_workbook(sheet)
|
||||
try:
|
||||
excel_text = " ".join(
|
||||
str(cell.value or "") for ws in workbook for row in ws for cell in row
|
||||
)
|
||||
finally:
|
||||
workbook.close()
|
||||
for text in (word_text, excel_text):
|
||||
assert "接口数据" in text
|
||||
assert "安卓业务后台" in text
|
||||
assert "actual-step" in text
|
||||
assert "用户指出业务状态仍未落库" in text
|
||||
|
||||
|
||||
def test_reports_share_lifecycle_and_distinguish_guidance_receipt_from_consumption(
|
||||
isolated, tmp_path, monkeypatch
|
||||
):
|
||||
from apps.admin_console.services import task_lifecycle
|
||||
from artemis.runtime import task_control
|
||||
|
||||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||||
job.update(
|
||||
status="completed",
|
||||
created_at=1700000010,
|
||||
worker_started_at=1700000020,
|
||||
ended_at=1700000120,
|
||||
finalized_at=1700000125,
|
||||
)
|
||||
isolated.save_job(job)
|
||||
monkeypatch.setattr(task_control, "TRACES_PATH", reports.TRACES_PATH)
|
||||
task_control.add_instruction(job["session_id"], "保留证据", instruction_id="unused-guidance")
|
||||
report = reports.session_report(job["session_id"])
|
||||
assert report["lifecycle"] == task_lifecycle.task_lifecycle(
|
||||
job["session_id"], db_path=isolated.path
|
||||
)
|
||||
assert report["lifecycle"]["durations"]["total"] == 115
|
||||
assert report["lifecycle"]["durations"]["execution"] is None
|
||||
assert report["instructions"][0]["status"] == "unapplied"
|
||||
word = tmp_path / "lifecycle.docx"
|
||||
excel = tmp_path / "lifecycle.xlsx"
|
||||
reports.word_report(report, word)
|
||||
reports.excel_report(report, excel)
|
||||
document = Document(word)
|
||||
word_text = "\n".join(p.text for p in document.paragraphs) + "\n".join(
|
||||
cell.text for table in document.tables for row in table.rows for cell in row.cells
|
||||
)
|
||||
book = load_workbook(excel)
|
||||
excel_text = "\n".join(str(cell.value) for sheet in book for row in sheet for cell in row)
|
||||
for content in (word_text, excel_text):
|
||||
assert "后台排队:10.0 秒" in content
|
||||
assert "执行与整理:未记录" in content
|
||||
assert "unused-guidance · 任务结束,未记录消费 · 接收" in content
|
||||
|
||||
|
||||
def test_video_export_accepts_finalized_worker_trace_and_rejects_external_path(
|
||||
isolated, tmp_path, monkeypatch
|
||||
):
|
||||
@@ -1188,6 +1294,7 @@ def test_video_export_accepts_finalized_worker_trace_and_rejects_external_path(
|
||||
second.write_bytes(b"second segment")
|
||||
|
||||
repository = Mock()
|
||||
repository.db_path = isolated.path
|
||||
repository.get_video_recording_for_session.return_value = {"local_video_path": str(first)}
|
||||
repository.get_session_by_id.return_value = {"video_filepath": str(first)}
|
||||
monkeypatch.setattr(reports, "SessionRepository", lambda: repository)
|
||||
|
||||
@@ -99,6 +99,7 @@ async def test_stop_requests_graceful_cancel_and_skips_kill_when_worker_exits():
|
||||
# The worker is asked to cancel itself; nothing is killed yet.
|
||||
terminate.assert_not_called()
|
||||
assert cancel_requests.is_cancel_requested(session_id="sess-1")
|
||||
assert cancel_requests.get_cancel_reason(session_id="sess-1").value == "user_stop"
|
||||
assert cancel_requests.is_cancel_requested(pid=24680, process_created_at=1234.5)
|
||||
assert len(TaskQueueService._forced_stop_tasks) == 1
|
||||
enforcer = next(iter(TaskQueueService._forced_stop_tasks))
|
||||
|
||||
@@ -180,6 +180,37 @@ def test_old_checker_config_does_not_enable_flash_review(monkeypatch, tmp_path):
|
||||
assert load_agent_config(path).flash.final_check_enabled is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enabled", [True, False])
|
||||
def test_saved_flash_switch_reaches_next_worker_without_restarting(
|
||||
isolated_settings, monkeypatch, enabled
|
||||
):
|
||||
from apps.admin_console.services.execution_config import execution_config
|
||||
from apps.admin_console.services.task_queue_service import TaskQueueService
|
||||
|
||||
monkeypatch.setattr("artemis.exploration.store.explorations.run", lambda _sid: None)
|
||||
monkeypatch.setattr("apps.admin_console.services.automation_store.store.job", lambda _sid: None)
|
||||
monkeypatch.setenv("ARTEMIS_FLASH_FINAL_CHECK_ENABLED", str(not enabled).lower())
|
||||
running = {"profile": "flash", "goal": "already running", "session_id": "running-review"}
|
||||
running["execution_config"] = execution_config(running)
|
||||
queued = {"profile": "flash", "goal": "queued", "session_id": "queued-review"}
|
||||
queued["execution_config"] = execution_config(queued)
|
||||
|
||||
service.save_capability_settings(service.CapabilitySettings(flash_final_check_enabled=enabled))
|
||||
# The real queue refreshes pending snapshots just before starting their worker.
|
||||
queued["execution_config"] = execution_config(queued)
|
||||
for item, expected in ((running, not enabled), (queued, enabled)):
|
||||
_cmd, env = TaskQueueService._build_worker_invocation(
|
||||
item,
|
||||
item["session_id"],
|
||||
item["session_id"],
|
||||
item["goal"],
|
||||
"flash",
|
||||
TaskQueueService._task_target(item),
|
||||
)
|
||||
assert env["ARTEMIS_FLASH_FINAL_CHECK_ENABLED"] == str(expected).lower()
|
||||
assert load_agent_config().pro.checker.final_check is True
|
||||
|
||||
|
||||
def test_screenshot_json_can_override_environment(monkeypatch, tmp_path):
|
||||
config_path = tmp_path / "artemis.jsonc"
|
||||
config_path.write_text(
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Shared task progress stays read-only and distinguishes receipt from consumption."""
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
import pytest
|
||||
|
||||
from apps.admin_console.routers import tasks
|
||||
from apps.admin_console.services import task_lifecycle as service
|
||||
from artemis.runtime import task_control as control
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def recorded(tmp_path, monkeypatch):
|
||||
db = tmp_path / "只读 progress.db"
|
||||
sid = "recorded-task"
|
||||
with sqlite3.connect(db) as conn:
|
||||
conn.executescript(
|
||||
"CREATE TABLE sessions (session_id TEXT, status TEXT, pid INTEGER, "
|
||||
"start_time REAL, end_time REAL);"
|
||||
"CREATE TABLE automation_jobs (session_id TEXT, payload TEXT);"
|
||||
"CREATE TABLE traces (session_id TEXT, type TEXT, payload TEXT);"
|
||||
)
|
||||
conn.execute("INSERT INTO sessions VALUES (?, 'completed', 17, 20, 120)", (sid,))
|
||||
conn.execute(
|
||||
"INSERT INTO automation_jobs VALUES (?, ?)",
|
||||
(
|
||||
sid,
|
||||
json.dumps(
|
||||
{
|
||||
"status": "completed",
|
||||
"created_at": 10,
|
||||
"worker_started_at": 20,
|
||||
"ended_at": 120,
|
||||
"finalized_at": 125,
|
||||
}
|
||||
),
|
||||
),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO traces VALUES (?, 'final_review_diagnostic', ?)",
|
||||
(sid, json.dumps({"started_at": 90, "ended_at": 110, "status": "completed"})),
|
||||
)
|
||||
monkeypatch.setattr(control, "TRACES_PATH", tmp_path / "traces")
|
||||
monkeypatch.setattr(service, "DB_PATH", db)
|
||||
control._write_atomic(
|
||||
control.control_dir(sid) / "task_control_state.json",
|
||||
{
|
||||
"pid": 17,
|
||||
"status": "running",
|
||||
"timing": {
|
||||
"started_at": 20,
|
||||
"state_started_at": 80,
|
||||
"seconds": {"queued": 10, "task_paused": 20, "running": 30},
|
||||
},
|
||||
},
|
||||
)
|
||||
return db, sid
|
||||
|
||||
|
||||
def test_finished_projection_is_stable_and_does_not_bootstrap_or_change_database(recorded):
|
||||
db, sid = recorded
|
||||
before = db.read_bytes()
|
||||
first = service.task_lifecycle(sid, db_path=str(db), now=500)
|
||||
assert first == service.task_lifecycle(sid, db_path=db, now=900)
|
||||
assert first["durations"] == {
|
||||
"total": 115,
|
||||
"queued": 10,
|
||||
"resource_wait": 10,
|
||||
"paused": 20,
|
||||
"final_review": 20,
|
||||
"execution": 50,
|
||||
"report_export": 5,
|
||||
}
|
||||
assert db.read_bytes() == before
|
||||
|
||||
|
||||
def test_legacy_guidance_array_and_detailed_web_receipt_share_terminal_state(recorded):
|
||||
_, sid = recorded
|
||||
pending = control.add_instruction(sid, "保存现有证据", instruction_id="pending")
|
||||
original = control.control_dir(sid) / "instructions" / "pending.json"
|
||||
before = original.read_bytes()
|
||||
app = FastAPI()
|
||||
app.include_router(tasks.router)
|
||||
client = TestClient(app)
|
||||
details = client.get(f"/api/tasks/{sid}/guidance?details=true").json()
|
||||
legacy = client.get(f"/api/tasks/{sid}/guidance").json()
|
||||
assert details["instructions"] == legacy
|
||||
assert legacy[0]["status"] == "unapplied"
|
||||
assert legacy[0]["accepted_at"] == pending["timestamp"]
|
||||
assert details["lifecycle"]["guidance_counts"]["unapplied"] == 1
|
||||
assert original.read_bytes() == before
|
||||
|
||||
|
||||
def test_state_from_another_worker_does_not_supply_current_timing(recorded):
|
||||
db, sid = recorded
|
||||
state = control.read_state(sid)
|
||||
state["pid"] = 18
|
||||
control._write_atomic(control.control_dir(sid) / "task_control_state.json", state)
|
||||
value = service.task_lifecycle(sid, db_path=db)
|
||||
assert not value["control_timing_recorded"]
|
||||
assert value["durations"]["execution"] is None
|
||||
assert value["durations"]["resource_wait"] is None
|
||||
|
||||
|
||||
def test_missing_database_is_not_created_and_does_not_hide_guidance(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(control, "TRACES_PATH", tmp_path)
|
||||
control.add_instruction("new-task", "等待安全检查点")
|
||||
path = tmp_path / "missing.db"
|
||||
value = service.task_lifecycle("new-task", db_path=path, now=100)
|
||||
assert value["phase"] == "unknown"
|
||||
assert value["guidance_counts"]["pending"] == 1
|
||||
assert not path.exists()
|
||||
@@ -12,6 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
|
||||
@@ -782,6 +783,33 @@ async def test_final_review_distinguishes_actual_delivery_from_report_claims():
|
||||
assert '"verified": false' in failed and "disk full" in failed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_final_review_receives_actual_source_ids_without_turning_backend_names_into_platforms():
|
||||
ctx = _mock_ctx()
|
||||
ctx.data_engine.get_agent_friendly_steps.return_value = [
|
||||
{
|
||||
"step_id": "real-step",
|
||||
"step_number": 7,
|
||||
"action_taken": {
|
||||
"plugin_id": "reader",
|
||||
"call_id": "real-call",
|
||||
"source_id": "real-source",
|
||||
"source": {"kind": "api", "target_id": "data-service", "label": "安卓业务后台"},
|
||||
},
|
||||
}
|
||||
]
|
||||
text = await _final_check_human_text(ctx)
|
||||
section = text.split(
|
||||
"# Recorded Source Inventory (provenance only, not verified assertions)\n", 1
|
||||
)[1].split("\n\n", 1)[0]
|
||||
value = json.loads(section)
|
||||
assert value["sources"][0]["kind"] == "api"
|
||||
assert value["sources"][0]["references"][0]["call_id"] == "real-call"
|
||||
guide = assemble_checker_prompt_segments("final", [], False)
|
||||
assert "does not by itself prove backend persistence" in guide
|
||||
assert "cannot silently overwrite the recorded result" in guide
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exploration_review_sees_conversation_and_can_read_earlier_messages():
|
||||
ctx = _mock_ctx()
|
||||
|
||||
@@ -72,6 +72,49 @@ def test_pc_first_and_intentional_interleaving_are_supported():
|
||||
assert progress.guard("cua-driver") is None
|
||||
|
||||
|
||||
def test_stage_updates_are_atomic_and_preserve_unmentioned_scope_and_order(tmp_path):
|
||||
progress = TaskProgress()
|
||||
resources = {"android", "cua-driver"}
|
||||
progress.update(plan(), resources=resources, directory=tmp_path)
|
||||
before = progress.value.model_dump()
|
||||
saved = (tmp_path / "notes/task_progress.md").read_bytes()
|
||||
with pytest.raises(ValidationError, match="完成或阻塞阶段必须记录"):
|
||||
progress.update(
|
||||
{"reason": "无证据不能完成", "stage_updates": [{"id": "phone", "status": "done"}]},
|
||||
resources=resources,
|
||||
directory=tmp_path,
|
||||
)
|
||||
assert progress.value.model_dump() == before
|
||||
assert (tmp_path / "notes/task_progress.md").read_bytes() == saved
|
||||
progress.update(
|
||||
{
|
||||
"reason": "手机已检查完",
|
||||
"stage_updates": [
|
||||
{"id": "phone", "status": "done", "evidence": "实际步骤记录 real-step"}
|
||||
],
|
||||
},
|
||||
resources=resources,
|
||||
directory=tmp_path,
|
||||
)
|
||||
assert progress.value.stages[1].model_dump() == before["stages"][1]
|
||||
assert [s.id for s in progress.value.stages] == ["phone", "pc"]
|
||||
assert progress.value.stages[0].scope == before["stages"][0]["scope"]
|
||||
progress.update(
|
||||
{"reason": "开始电脑检查", "stage_updates": [{"id": "pc", "status": "active"}]},
|
||||
resources=resources,
|
||||
)
|
||||
assert progress.guard("cua-driver") is None
|
||||
with pytest.raises(ValueError, match="未知 id"):
|
||||
progress.update(
|
||||
{"reason": "错拼不能覆盖", "stage_updates": [{"id": "typo", "status": "done"}]},
|
||||
resources=resources,
|
||||
)
|
||||
with pytest.raises(ValidationError, match="只能提供一种"):
|
||||
progress.update(
|
||||
{**plan(), "stage_updates": [{"id": "pc", "status": "active"}]}, resources=resources
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flash_preflight_does_not_authorize_pc_mutation(monkeypatch):
|
||||
plugins = SimpleNamespace(
|
||||
|
||||
@@ -19,6 +19,7 @@ from artemis.agents.operator.operator import OperatorNode
|
||||
from artemis.config.agent import MemoryTranscriptConfig
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.core.tool_failure import ToolFailure
|
||||
from artemis.utils.action_target import observation_anchor
|
||||
import pytest
|
||||
|
||||
# These tests exercise the legacy 2-message prompt path. Since M5 the
|
||||
@@ -126,6 +127,8 @@ async def test_perform_action_validation():
|
||||
"target_bounds": "[100,100][300,500]",
|
||||
"target_resource_id": "btn_1",
|
||||
"target_class": "android.widget.Button",
|
||||
"target_index": 1,
|
||||
**observation_anchor(mock_state),
|
||||
}
|
||||
]
|
||||
|
||||
@@ -156,6 +159,8 @@ async def test_perform_action_validation():
|
||||
"target_bounds": "[300,400][500,600]",
|
||||
"target_resource_id": "input_1",
|
||||
"target_class": "android.widget.EditText",
|
||||
"target_index": 2,
|
||||
**observation_anchor(mock_state),
|
||||
}
|
||||
|
||||
# Test swipe direction
|
||||
@@ -184,6 +189,8 @@ async def test_perform_action_validation():
|
||||
"target_bounds": "[300,400][500,600]",
|
||||
"target_resource_id": "input_1",
|
||||
"target_class": "android.widget.EditText",
|
||||
"target_index": 2,
|
||||
**observation_anchor(mock_state),
|
||||
}
|
||||
]
|
||||
|
||||
@@ -290,6 +297,7 @@ async def test_coordinate_click_requires_and_records_the_models_description():
|
||||
"times": 1,
|
||||
"delay_ms": 100,
|
||||
"target_description": "play button",
|
||||
**observation_anchor(mock_state),
|
||||
}
|
||||
]
|
||||
|
||||
@@ -883,6 +891,8 @@ async def test_long_press_action_translation():
|
||||
"target_bounds": "[100,100][300,500]",
|
||||
"target_resource_id": "btn_1",
|
||||
"target_class": "android.widget.Button",
|
||||
"target_index": 1,
|
||||
**observation_anchor(mock_state),
|
||||
}
|
||||
]
|
||||
|
||||
@@ -910,6 +920,7 @@ async def test_long_press_action_translation():
|
||||
"normalized_coordinates": [500, 600],
|
||||
"duration": 1000,
|
||||
"target_description": "app icon",
|
||||
**observation_anchor(mock_state),
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@@ -15,7 +15,12 @@
|
||||
"""Tests for pixel safety-net target descriptions."""
|
||||
|
||||
from pathlib import Path
|
||||
from io import BytesIO
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import AsyncMock
|
||||
import base64
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from artemis.agents.validator import precondition_pixel as pp
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
@@ -23,6 +28,58 @@ from langchain_core.messages import HumanMessage, SystemMessage
|
||||
_PNG = b"fake-image-bytes"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"response",
|
||||
[
|
||||
{"is_present": True, "confidence": 0.99, "target_kind": "surface", "target_matches": True},
|
||||
{"is_present": True, "confidence": 0.99},
|
||||
{"is_present": True, "confidence": 0.2, "target_kind": "control", "target_matches": True},
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"target",
|
||||
[
|
||||
{"action": "tap", "target_text": "Input"},
|
||||
{"action": "input_text", "target_description": "Input"},
|
||||
],
|
||||
)
|
||||
async def test_named_input_is_never_approved_as_blank_or_unverified(response, target, monkeypatch):
|
||||
monkeypatch.setattr(pp.visualization, "crop_and_annotate_target", lambda *a, **k: _PNG)
|
||||
monkeypatch.setattr(pp, "acomplete_structured", AsyncMock(return_value=response))
|
||||
session = AsyncMock()
|
||||
session.screenshot_b64.return_value = base64.b64encode(_PNG).decode()
|
||||
verdict = await pp._run_attempt(session, None, "prompt", _PNG, [1, 2], target, None, 1)
|
||||
assert verdict[0] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_renaming_rejected_point_cannot_dispatch_again(monkeypatch):
|
||||
monkeypatch.setattr(pp, "_init_llm_and_prompt", lambda *a: (None, "prompt"))
|
||||
attempt = AsyncMock(
|
||||
return_value=(False, pp.ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED, "No input")
|
||||
)
|
||||
monkeypatch.setattr(pp, "_run_attempt", attempt)
|
||||
ctx = MagicMock()
|
||||
image = Image.new("RGB", (400, 400), "white")
|
||||
frames = []
|
||||
for color in ("black", "blue"):
|
||||
image.putpixel((5, 5), Image.new("RGB", (1, 1), color).getpixel((0, 0)))
|
||||
encoded = BytesIO()
|
||||
image.save(encoded, format="PNG")
|
||||
frames.append(base64.b64encode(encoded.getvalue()).decode())
|
||||
for description, frame in zip(("pairing input", "blank surface"), frames, strict=True):
|
||||
verdict = await pp.validate_action_precondition_pixel(
|
||||
ctx,
|
||||
AsyncMock(),
|
||||
{"action": "tap", "coordinates": [200, 300], "target_description": description},
|
||||
frame,
|
||||
get_llm_fn=lambda: None,
|
||||
)
|
||||
assert verdict[0] is False
|
||||
assert attempt.await_count == 1
|
||||
|
||||
|
||||
def _texts(messages: list) -> list[str]:
|
||||
human = messages[1]
|
||||
return [part["text"] for part in human.content if part["type"] == "text"]
|
||||
|
||||
@@ -123,11 +123,19 @@ async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that ValidatorNode executes actions successfully."""
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
state.initial_goal = "在真实审批表单上 Approve once,不要普通聊天代答或重复提交。"
|
||||
|
||||
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
|
||||
with (
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
|
||||
patch(
|
||||
"artemis.services.llm.get_llm",
|
||||
side_effect=AssertionError("Unexpected AI action review"),
|
||||
) as get_llm,
|
||||
):
|
||||
node = ValidatorNode(mock_context)
|
||||
result = await node(state)
|
||||
|
||||
get_llm.assert_not_called()
|
||||
assert "last_execution_result" in result
|
||||
report = result["last_execution_result"]
|
||||
assert "execution" in report
|
||||
@@ -144,10 +152,40 @@ async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
|
||||
# tests/unit/mcp/test_action_session.py against the in-process ActionSession.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pro_unconfirmed_input_stops_burst_before_submission(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
mock_mcp.hierarchy = [
|
||||
{
|
||||
"class": "android.widget.EditText",
|
||||
"resource_id": "fixture:input",
|
||||
"text": "Paste here",
|
||||
"hint": "Paste here",
|
||||
"showing_hint": True,
|
||||
"focused": True,
|
||||
"bounds": "[100,200][500,400]",
|
||||
}
|
||||
]
|
||||
decisions = [
|
||||
{"action": "focus_and_input_text", "coordinates": [300, 300], "text": "fixture value"},
|
||||
{"action": "tap", "coordinates": [300, 500]},
|
||||
]
|
||||
state = DummyState(
|
||||
structured_decisions=json.dumps(decisions), latest_screenshot=temp_screenshot
|
||||
)
|
||||
with patch("artemis.utils.image_diff.check_ui_change", return_value=False):
|
||||
result = await ValidatorNode(mock_context)(state)
|
||||
report = result["last_execution_result"]
|
||||
assert report["status"] == "failed"
|
||||
assert report["execution"][0]["input_verification"] == "empty"
|
||||
assert len(mock_mcp.calls_for("input_text")) == 1
|
||||
assert not mock_mcp.calls_for("click")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_exec_error_opens_incident(mock_mcp, mock_context, temp_screenshot):
|
||||
"""A vetted single action that the device rejects is retried once, then an
|
||||
execution incident is opened for the Operator (no repair agent)."""
|
||||
"""An uncertain device failure opens an incident without replaying the write."""
|
||||
mock_mcp.action_handler = lambda name, args: (
|
||||
(False, "Error: Element not found") if name == "click" else (True, "")
|
||||
)
|
||||
@@ -165,8 +203,8 @@ async def test_validator_exec_error_opens_incident(mock_mcp, mock_context, temp_
|
||||
assert len(report["execution"]) == 1
|
||||
assert report["execution"][0]["attempts"] == [
|
||||
"Error: Element not found",
|
||||
"Error: Element not found",
|
||||
]
|
||||
assert len(mock_mcp.calls_for("click")) == 1
|
||||
assert "repair" not in report["execution"][0]
|
||||
|
||||
incident = report["incident"]
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
from copy import deepcopy
|
||||
|
||||
from artemis.data_engine.evidence_sources import recorded_sources
|
||||
|
||||
|
||||
def record(kind, target, **extra):
|
||||
return {
|
||||
"plugin_id": "adapter",
|
||||
"source": {"kind": kind, "target_id": target, "label": "安卓业务后台"},
|
||||
**extra,
|
||||
}
|
||||
|
||||
|
||||
def test_business_backend_name_does_not_become_an_execution_platform_or_assertion():
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step-real-id",
|
||||
"step_number": 7,
|
||||
"last_execution_result": {
|
||||
"execution": [record("api", "backend", call_id="call-real-id", read_only=True)]
|
||||
},
|
||||
}
|
||||
]
|
||||
before = deepcopy(steps)
|
||||
value = recorded_sources(steps)
|
||||
source = value["sources"][0]
|
||||
assert source["kind"] == "api"
|
||||
assert source["category"] == "auxiliary"
|
||||
assert source["references"] == [
|
||||
{"step_id": "step-real-id", "step_number": 7, "call_id": "call-real-id", "read_only": True}
|
||||
]
|
||||
assert not any(key in source for key in ("passed", "verified", "verdict"))
|
||||
assert steps == before
|
||||
|
||||
|
||||
def test_sources_remain_separate_and_references_use_host_ids():
|
||||
steps = [
|
||||
{
|
||||
"step_id": "s1",
|
||||
"step_number": 1,
|
||||
"action_taken": record("computer", "window", call_id="c1"),
|
||||
},
|
||||
{
|
||||
"step_id": "s2",
|
||||
"step_number": 2,
|
||||
"action_taken": record("computer", "window", call_id="c2"),
|
||||
},
|
||||
{
|
||||
"step_id": "s3",
|
||||
"step_number": 3,
|
||||
"action_taken": record("android", "phone", call_id="c3"),
|
||||
},
|
||||
]
|
||||
value = recorded_sources(steps)
|
||||
assert value["total_sources"] == 2
|
||||
assert value["sources"][0]["reference_count"] == 2
|
||||
assert [r["call_id"] for r in value["sources"][0]["references"]] == ["c1", "c2"]
|
||||
assert value["sources"][0]["id"] == recorded_sources(steps[:1])["sources"][0]["id"]
|
||||
assert value["sources"][1]["category"] == "interface"
|
||||
|
||||
|
||||
def test_model_arguments_and_unsourced_images_do_not_fabricate_source_metadata():
|
||||
value = recorded_sources(
|
||||
[
|
||||
{
|
||||
"step_id": "s",
|
||||
"step_number": 3,
|
||||
"pre_image_name": "hash",
|
||||
"action_taken": {"args": {"source": {"kind": "computer", "target_id": "claim"}}},
|
||||
}
|
||||
]
|
||||
)
|
||||
assert value["sources"] == []
|
||||
assert value["unattributed_references"] == [{"step_id": "s", "step_number": 3}]
|
||||
|
||||
|
||||
def test_summary_is_bounded_and_reports_omissions_explicitly():
|
||||
steps = [{"step_id": str(i), "action_taken": record("file", str(i))} for i in range(10)]
|
||||
value = recorded_sources(steps, limit=2)
|
||||
assert len(value["sources"]) == 2
|
||||
assert value["total_sources"] == 10
|
||||
assert value["sources_omitted"] == 8
|
||||
repeated = recorded_sources(
|
||||
[{"step_id": str(i), "action_taken": record("api", "one")} for i in range(20)]
|
||||
)
|
||||
assert len(repeated["sources"][0]["references"]) == 8
|
||||
assert repeated["sources"][0]["reference_count"] == 20
|
||||
|
||||
|
||||
def test_partial_source_metadata_does_not_hide_unknown_actions_in_the_same_step():
|
||||
value = recorded_sources(
|
||||
[
|
||||
{
|
||||
"step_id": "mixed",
|
||||
"last_execution_result": {
|
||||
"execution": [
|
||||
record("api", "service"),
|
||||
{"action": "tap"},
|
||||
]
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
assert value["total_sources"] == 1
|
||||
assert value["unattributed_step_count"] == 1
|
||||
@@ -8,11 +8,13 @@ action), and report their status explicitly instead of sniffing the text for
|
||||
the word "Error".
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.mcp.action_executor import AGENT_TOOL_NAMES, McpActionExecutor
|
||||
from artemis.mcp.action_types import ActionCode, ActionResult
|
||||
from artemis.tools.explorer_tool import ExplorerCandidate, ExplorerOutcome
|
||||
|
||||
|
||||
@@ -48,6 +50,61 @@ def test_helper_tools_are_not_device_actions():
|
||||
assert not (AGENT_TOOL_NAMES & executor.action_tool_names)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("extension", [False, True])
|
||||
@pytest.mark.parametrize("ok", [False, True])
|
||||
async def test_authorized_actions_dispatch_without_secondary_ai_review(extension, ok):
|
||||
from mcp.types import CallToolResult, TextContent
|
||||
|
||||
executor = _make_executor(agent_name="flash")
|
||||
state = _state()
|
||||
state.initial_goal = (
|
||||
"仅发送一次指定的问题。等待真实审批表单,Approve once (not remember rule)。"
|
||||
"不要普通聊天代答或重复提交;不向该测试另发消息、不停止任务,也不操作其他会话。"
|
||||
)
|
||||
name = "submit_form" if extension else "click"
|
||||
args = (
|
||||
{"window_id": 123}
|
||||
if extension
|
||||
else {
|
||||
"target": [500, 500],
|
||||
"target_description": "Approve",
|
||||
}
|
||||
)
|
||||
executor.actuator.extensions.return_value = [SimpleNamespace(name="submit_form")]
|
||||
session = SimpleNamespace(
|
||||
call=AsyncMock(
|
||||
return_value=ActionResult(
|
||||
ok=ok,
|
||||
code=ActionCode.OK if ok else ActionCode.DEVICE_ERROR,
|
||||
action=name,
|
||||
message="tool outcome",
|
||||
)
|
||||
),
|
||||
call_raw=AsyncMock(
|
||||
return_value=CallToolResult(
|
||||
isError=not ok, content=[TextContent(type="text", text="tool outcome")]
|
||||
)
|
||||
),
|
||||
)
|
||||
executor._session_or_start = AsyncMock(return_value=session)
|
||||
executor._observe_after_action = AsyncMock(return_value=(SimpleNamespace(ok=False), None))
|
||||
|
||||
with patch(
|
||||
"artemis.services.llm.get_llm", side_effect=AssertionError("Unexpected AI action review")
|
||||
) as get_llm:
|
||||
result = await executor.execute(name, args, "action-1", state)
|
||||
|
||||
get_llm.assert_not_called()
|
||||
wire_args = args if extension else {"target": [500, 500], "times": 1, "delay_ms": 100}
|
||||
dispatched = session.call_raw if extension else session.call
|
||||
unused = session.call if extension else session.call_raw
|
||||
dispatched.assert_awaited_once_with(name, wire_args)
|
||||
unused.assert_not_awaited()
|
||||
assert result.status == ("success" if ok else "error")
|
||||
assert result.text_summary == "tool outcome"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flash_saves_and_reads_text_with_verified_receipt_without_device_actions(tmp_path):
|
||||
import hashlib
|
||||
|
||||
@@ -27,6 +27,7 @@ from unittest.mock import Mock
|
||||
import pytest
|
||||
|
||||
from artemis.mcp.action_executor import McpActionExecutor, _ArgError
|
||||
from artemis.utils.action_target import observation_anchor
|
||||
|
||||
|
||||
def _make_executor(width=1080, height=2400):
|
||||
@@ -90,6 +91,8 @@ def test_click_index_resolves_to_the_elements_center_and_records_observed_fields
|
||||
"target_bounds": [37, 480, 963, 560],
|
||||
"target_resource_id": "android:id/title",
|
||||
"target_class": "android.widget.TextView",
|
||||
"target_index": 1,
|
||||
**observation_anchor(_indexed_state()),
|
||||
}
|
||||
assert "target_description" not in recorded
|
||||
|
||||
|
||||
@@ -384,7 +384,10 @@ def test_mobile_manage_task_inject_instruction(temp_trace_env):
|
||||
ok_res = mobile_manage_task(
|
||||
action="inject_instruction", trace_id=trace_id, instruction="Scroll down slowly"
|
||||
)
|
||||
assert "current model step interrupted" in ok_res["message"]
|
||||
assert "next safe checkpoint" in ok_res["message"]
|
||||
assert "not guaranteed" in ok_res["message"]
|
||||
assert ok_res["guidance"]["id"]
|
||||
assert ok_res["guidance"]["status"] == "pending"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -476,11 +479,33 @@ async def test_mobile_inspect_trace_includes_device_serial(temp_trace_env):
|
||||
"CREATE TABLE traces (trace_id TEXT PRIMARY KEY, step_id TEXT, session_id TEXT, type TEXT, name TEXT, status TEXT, timestamp REAL, duration REAL, payload TEXT)"
|
||||
)
|
||||
conn.execute("INSERT INTO sessions VALUES (?, ?, ?)", (trace_id, 1000.0, None))
|
||||
conn.executemany(
|
||||
"INSERT INTO steps (step_id,session_id,step_number,timestamp) VALUES (?,?,?,?)",
|
||||
[(f"actual-{n}", trace_id, n, 1000 + n) for n in range(1, 4)],
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
res_summary = await mobile_inspect_trace(action="view_summary", trace_id=trace_id)
|
||||
assert "**Device Serial:** `pixel-target-999`" in res_summary
|
||||
res_summary = await mobile_inspect_trace(action="view_summary", trace_id=trace_id, limit=2)
|
||||
assert "**Device Serial:** `pixel-target-999`" in res_summary.content[0].text
|
||||
assert res_summary.content[0].text.startswith("# ARTEMIS Flash Execution Summary")
|
||||
assert res_summary.structuredContent["steps"] == {
|
||||
"items": [
|
||||
{"step_id": "actual-1", "step_number": 1},
|
||||
{"step_id": "actual-2", "step_number": 2},
|
||||
],
|
||||
"total": 3,
|
||||
"offset": 0,
|
||||
"limit": 2,
|
||||
"next_offset": 2,
|
||||
}
|
||||
last_page = await mobile_inspect_trace(
|
||||
action="view_summary", trace_id=trace_id, offset=2, limit=2
|
||||
)
|
||||
assert last_page.structuredContent["steps"]["next_offset"] is None
|
||||
assert last_page.structuredContent["steps"]["items"] == [
|
||||
{"step_id": "actual-3", "step_number": 3}
|
||||
]
|
||||
|
||||
|
||||
def test_trace_store_device_serial(temp_trace_env):
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Protocol regressions without starting ARTEMIS or invoking a device."""
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from mcp.server.fastmcp.exceptions import ToolError
|
||||
from mcp.shared.memory import create_connected_server_and_client_session
|
||||
import pytest
|
||||
|
||||
from apps.admin_console.schemas.task_schema import RunRequest
|
||||
from artemis.mcp.tool_server import ArtemisFastMCP
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_arguments_are_rejected_before_dispatch():
|
||||
mcp = ArtemisFastMCP("strict-input-test", strict_inputs=True)
|
||||
dispatched = []
|
||||
|
||||
@mcp.tool()
|
||||
def submit(task_desc: str) -> dict:
|
||||
dispatched.append(task_desc)
|
||||
return {"ok": True}
|
||||
|
||||
with pytest.raises(ToolError, match="未知参数.*goal"):
|
||||
await mcp.call_tool("submit", {"task_desc": "original", "goal": "silently replace?"})
|
||||
assert dispatched == []
|
||||
assert (await mcp.list_tools())[0].inputSchema["additionalProperties"] is False
|
||||
await mcp.call_tool("submit", {"task_desc": "accepted"})
|
||||
assert dispatched == ["accepted"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_legacy_json_has_structured_content_without_changing_text():
|
||||
mcp = ArtemisFastMCP("structured-test", strict_inputs=True)
|
||||
|
||||
@mcp.tool()
|
||||
def legacy() -> Any:
|
||||
return {"items": [{"step_id": "actual-step"}], "offset": 0, "total": 1, "next_offset": None}
|
||||
|
||||
content, structured = await mcp.call_tool("legacy", {})
|
||||
assert json.loads(content[0].text) == structured
|
||||
assert structured["items"][0]["step_id"] == "actual-step"
|
||||
async with create_connected_server_and_client_session(mcp) as session:
|
||||
wire = await session.call_tool("legacy", {})
|
||||
assert wire.structuredContent == structured
|
||||
assert json.loads(wire.content[0].text) == structured
|
||||
rejected = await session.call_tool("legacy", {"goal": "unknown field"})
|
||||
assert rejected.isError is True
|
||||
|
||||
|
||||
def test_http_submission_rejects_unknown_and_misspelled_fields():
|
||||
from pydantic import ValidationError
|
||||
|
||||
with pytest.raises(ValidationError, match="extra_forbidden"):
|
||||
RunRequest.model_validate({"goal": "original", "task_desc": "wrong contract"})
|
||||
assert (
|
||||
RunRequest.model_validate({"goal": "accepted", "runtime_overrides": {}}).goal == "accepted"
|
||||
)
|
||||
@@ -19,6 +19,7 @@ import time
|
||||
import pytest
|
||||
|
||||
from artemis.runtime import cancel_requests as cr
|
||||
from artemis.runtime.cancellation import CancellationReason
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
@@ -91,3 +92,27 @@ async def test_watch_delivers_once_and_clears_marker():
|
||||
assert await asyncio.wait_for(watcher, timeout=2) is True
|
||||
assert delivered == [1]
|
||||
assert not cr.is_cancel_requested(session_id="w-1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("cause", [CancellationReason.USER_STOP, CancellationReason.UNKNOWN])
|
||||
async def test_watch_forwards_only_the_structured_cause(cause):
|
||||
path = cr.request_cancel(
|
||||
session_id="with-reason", reason="private stop details", cancel_reason=cause
|
||||
)[0]
|
||||
delivered = []
|
||||
assert await cr.watch_for_cancel_request(
|
||||
delivered.append, session_id="with-reason", pid=0, pass_reason=True
|
||||
)
|
||||
assert delivered == [cause]
|
||||
assert not path.exists()
|
||||
|
||||
|
||||
def test_legacy_or_unrecognized_marker_reasons_remain_unknown():
|
||||
path = cr.request_cancel(session_id="legacy", reason="stop from UI")[0]
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
for value in [None, "private user instruction", ["user_stop"]]:
|
||||
payload["cancel_reason"] = value
|
||||
path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
assert cr.get_cancel_reason(session_id="legacy") is CancellationReason.UNKNOWN
|
||||
assert cr.is_cancel_requested(session_id="legacy")
|
||||
@@ -14,6 +14,47 @@ import pytest
|
||||
import artemis.runtime.task_control as task_control
|
||||
|
||||
|
||||
def test_transition_timing_keeps_repeated_states_and_separates_pause_and_resource_wait(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
||||
clock = [10]
|
||||
monkeypatch.setattr(
|
||||
task_control,
|
||||
"time",
|
||||
SimpleNamespace(**{**vars(task_control.time), "time": lambda: clock[0]}),
|
||||
)
|
||||
for at, status in (
|
||||
(10, "queued"),
|
||||
(20, "running"),
|
||||
(25, "running"),
|
||||
(30, "task_paused"),
|
||||
(40, "resuming"),
|
||||
(50, "running"),
|
||||
):
|
||||
clock[0] = at
|
||||
task_control.write_control_state("timed-task", status)
|
||||
assert task_control.read_state("timed-task")["timing"] == {
|
||||
"started_at": 10,
|
||||
"state_started_at": 50,
|
||||
"seconds": {"queued": 10, "running": 10, "task_paused": 10, "resuming": 10},
|
||||
}
|
||||
|
||||
|
||||
def test_non_object_control_state_and_invalid_timing_can_be_replaced(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
||||
path = task_control.control_dir("broken-state") / "task_control_state.json"
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text("[]", encoding="utf-8")
|
||||
assert task_control.read_state("broken-state") == {}
|
||||
task_control.write_control_state("broken-state", "running")
|
||||
state = task_control.read_state("broken-state")
|
||||
state["timing"] = "invalid"
|
||||
task_control._write_atomic(path, state)
|
||||
task_control.write_control_state("broken-state", "task_paused")
|
||||
assert task_control.read_state("broken-state")["timing"]["seconds"] == {}
|
||||
|
||||
|
||||
def _windows_access_error(code: int) -> PermissionError:
|
||||
error = PermissionError(errno.EACCES, "state file is in use")
|
||||
error.winerror = code
|
||||
@@ -423,6 +464,28 @@ def test_guidance_history_distinguishes_submission_from_runner_consumption(tmp_p
|
||||
assert all(item["status"] == "consumed" and item["consumed_at"] for item in history)
|
||||
|
||||
|
||||
def test_terminal_guidance_reports_no_consumption_and_keeps_original_receipt(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
||||
receipt = task_control.add_instruction("task-123", "检查新入口", instruction_id="guidance")
|
||||
history = task_control.list_instructions("task-123", task_status="cancelled")
|
||||
assert history[0]["status"] == "unapplied"
|
||||
assert history[0]["accepted_at"] == receipt["timestamp"]
|
||||
assert "consumed_at" not in history[0]
|
||||
assert task_control.list_instructions("task-123")[0]["status"] == "pending"
|
||||
|
||||
|
||||
def test_claimed_guidance_is_visible_and_not_falsely_reported_as_consumed(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
||||
task_control.add_instruction("task-123", "检查新入口", instruction_id="guidance")
|
||||
directory = task_control.control_dir("task-123") / "instructions"
|
||||
(directory / "guidance.json").rename(directory / "guidance.reading")
|
||||
assert task_control.list_instructions("task-123")[0]["status"] == "reading"
|
||||
assert (
|
||||
task_control.list_instructions("task-123", task_status="completed")[0]["status"]
|
||||
== "unconfirmed"
|
||||
)
|
||||
|
||||
|
||||
def test_guidance_history_endpoint_returns_pending_and_consumed(tmp_path, monkeypatch):
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
from artemis.runtime.task_progress import progress_snapshot
|
||||
|
||||
|
||||
def test_queued_time_is_not_execution_time_even_with_legacy_started_at():
|
||||
value = progress_snapshot(job={"status": "queued", "created_at": 10, "started_at": 10}, now=50)
|
||||
assert value["phase"] == "queued"
|
||||
assert value["durations"]["queued"] == 40
|
||||
assert value["durations"]["execution"] is None
|
||||
|
||||
|
||||
def test_progress_splits_wait_pause_review_and_export_without_double_counting():
|
||||
value = progress_snapshot(
|
||||
job={
|
||||
"status": "completed",
|
||||
"created_at": 10,
|
||||
"worker_started_at": 20,
|
||||
"ended_at": 120,
|
||||
"finalized_at": 125,
|
||||
},
|
||||
control={
|
||||
"status": "running",
|
||||
"timing": {
|
||||
"started_at": 20,
|
||||
"state_started_at": 80,
|
||||
"seconds": {"queued": 10, "task_paused": 20, "resuming": 10, "running": 20},
|
||||
},
|
||||
},
|
||||
reviews=[{"started_at": 90, "ended_at": 110}, {"started_at": 100, "ended_at": 115}],
|
||||
now=500,
|
||||
)
|
||||
assert value["phase"] == "finished"
|
||||
assert value["durations"] == {
|
||||
"total": 115,
|
||||
"queued": 10,
|
||||
"resource_wait": 20,
|
||||
"paused": 20,
|
||||
"final_review": 25,
|
||||
"execution": 35,
|
||||
"report_export": 5,
|
||||
}
|
||||
|
||||
|
||||
def test_active_review_has_its_own_phase_and_terminal_review_is_not_still_active():
|
||||
review = {"started_at": 30, "status": "running", "updated_at": 35}
|
||||
live = progress_snapshot(job={"status": "running"}, reviews=[review], now=40)
|
||||
assert live["phase"] == "final_review"
|
||||
assert live["durations"]["final_review"] == 10
|
||||
stopped = progress_snapshot(
|
||||
job={"status": "cancelled", "ended_at": 45}, reviews=[review], now=500
|
||||
)
|
||||
assert stopped["phase"] == "finished"
|
||||
assert stopped["durations"]["final_review"] == 5
|
||||
assert not stopped["review_timing_complete"]
|
||||
|
||||
|
||||
def test_legacy_missing_intervals_are_unknown_rather_than_zero_or_model_speed():
|
||||
value = progress_snapshot(
|
||||
session={"status": "completed", "start_time": 20, "end_time": 100}, now=500
|
||||
)
|
||||
assert value["durations"]["queued"] is None
|
||||
assert value["durations"]["execution"] is None
|
||||
assert value["durations"]["paused"] is None
|
||||
assert value["durations"]["total"] is None
|
||||
assert value["observed_seconds"] == 80
|
||||
|
||||
|
||||
def test_waiting_for_resource_and_pause_are_not_running():
|
||||
for status, phase in (
|
||||
("queued", "waiting_resources"),
|
||||
("resuming", "waiting_resources"),
|
||||
("task_paused", "paused"),
|
||||
):
|
||||
value = progress_snapshot(
|
||||
job={"status": "running", "worker_started_at": 10}, control={"status": status}, now=50
|
||||
)
|
||||
assert value["phase"] == phase
|
||||
|
||||
|
||||
def test_terminal_without_end_timestamp_does_not_keep_increasing_or_guess_duration():
|
||||
job = {"status": "completed", "created_at": 10, "worker_started_at": 20}
|
||||
first = progress_snapshot(job=job, now=100)
|
||||
assert first == progress_snapshot(job=job, now=500)
|
||||
assert first["observed_at"] is None
|
||||
assert all(first["durations"][key] is None for key in ("total", "execution", "report_export"))
|
||||
|
||||
|
||||
def test_export_in_progress_keeps_total_current_without_extending_execution():
|
||||
value = progress_snapshot(
|
||||
job={"status": "finalizing", "created_at": 10, "worker_started_at": 100, "ended_at": 200},
|
||||
now=230,
|
||||
)
|
||||
assert value["phase"] == "report_export"
|
||||
assert value["observed_at"] == 230
|
||||
assert value["durations"]["total"] == 220
|
||||
assert value["durations"]["report_export"] == 30
|
||||
assert value["worker_ended_at"] == 200
|
||||
@@ -19,11 +19,13 @@ import asyncio
|
||||
import pytest
|
||||
|
||||
from artemis.runtime import cancel_requests
|
||||
from artemis.runtime.cancellation import CancellationReason
|
||||
from artemis.sdk.agent import Agent
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_external_cancel_marker_cancels_running_task(tmp_path, monkeypatch):
|
||||
@pytest.mark.parametrize("cause", [CancellationReason.USER_STOP, CancellationReason.UNKNOWN])
|
||||
async def test_external_cancel_marker_cancels_running_task(tmp_path, monkeypatch, cause):
|
||||
monkeypatch.setattr(cancel_requests, "get_temp_dir", lambda _subfolder=None: tmp_path / "c")
|
||||
monkeypatch.setenv("ARTEMIS_CANCEL_POLL_SECONDS", "0.05")
|
||||
|
||||
@@ -38,9 +40,12 @@ async def test_external_cancel_marker_cancels_running_task(tmp_path, monkeypatch
|
||||
await asyncio.sleep(0.15)
|
||||
assert not agent._current_task.done()
|
||||
|
||||
cancel_requests.request_cancel(session_id="sess-cancel", reason="stop from UI")
|
||||
cancel_requests.request_cancel(
|
||||
session_id="sess-cancel", reason="stop from UI", cancel_reason=cause
|
||||
)
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
with pytest.raises(asyncio.CancelledError) as error:
|
||||
await asyncio.wait_for(agent._current_task, timeout=2)
|
||||
assert error.value.args == (cause,)
|
||||
await asyncio.wait_for(watcher, timeout=2)
|
||||
assert not cancel_requests.is_cancel_requested(session_id="sess-cancel")
|
||||
@@ -0,0 +1,225 @@
|
||||
"""Cancellation provenance must survive the real runtime-to-model path."""
|
||||
|
||||
import asyncio
|
||||
from copy import deepcopy
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.llm.final_review import FinalReviewBudget
|
||||
from artemis.llm.reliability import LLMExhaustedError
|
||||
from artemis.runtime.cancellation import CancellationReason
|
||||
from artemis.runtime import task_control
|
||||
from artemis.sdk.agent import Agent
|
||||
from artemis.services import llm
|
||||
|
||||
|
||||
class Engine:
|
||||
current_session_id = "cancellation-test"
|
||||
current_step_id = None
|
||||
|
||||
def __init__(self):
|
||||
self.records = []
|
||||
|
||||
def record_trace(self, **record):
|
||||
self.records.append(deepcopy(record))
|
||||
|
||||
def _publish(self, *_):
|
||||
pass
|
||||
|
||||
@property
|
||||
def attempt(self):
|
||||
return next(r["payload"] for r in reversed(self.records) if r["type"] == "llm_diagnostic")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guidance_cancellation_reaches_request_without_retry(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
||||
engine = Engine()
|
||||
started = asyncio.Event()
|
||||
calls = 0
|
||||
|
||||
async def model():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
started.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
control = task_control.TaskControl("task", object())
|
||||
review = FinalReviewBudget(
|
||||
engine, attempt_id="final#1", request_timeout=10, max_retries=5, retry_delay=0
|
||||
)
|
||||
running = asyncio.create_task(
|
||||
control.run_interruptibly(
|
||||
review.run(lambda: llm._run_with_recovery(model, max_retries=5, engine=engine))
|
||||
)
|
||||
)
|
||||
await asyncio.wait_for(started.wait(), 2)
|
||||
task_control.add_instruction("task", "Use the newly observed page")
|
||||
assert await asyncio.wait_for(running, 2) == (True, None)
|
||||
assert engine.attempt["phase"] == "cancelled"
|
||||
assert engine.attempt["cancel_reason"] == "guidance_replaced"
|
||||
assert review.data["status"] == "cancelled"
|
||||
assert review.data["cancel_reason"] == "guidance_replaced"
|
||||
assert calls == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sdk_stop_reaches_request_without_retry():
|
||||
engine = Engine()
|
||||
started = asyncio.Event()
|
||||
|
||||
async def model():
|
||||
started.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
agent = Agent.__new__(Agent)
|
||||
agent._current_task = asyncio.create_task(
|
||||
llm._run_with_recovery(model, max_retries=5, engine=engine)
|
||||
)
|
||||
await asyncio.wait_for(started.wait(), 2)
|
||||
agent.stop_current_task(reason=CancellationReason.USER_STOP)
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await agent._current_task
|
||||
assert engine.attempt["phase"] == "cancelled"
|
||||
assert engine.attempt["cancel_reason"] == "user_stop"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_outer_timeout_is_not_a_request_timeout():
|
||||
engine = Engine()
|
||||
|
||||
async def model():
|
||||
await asyncio.Event().wait()
|
||||
|
||||
with pytest.raises(TimeoutError):
|
||||
await llm.invoke_llm_with_timeout_message(
|
||||
llm._run_with_recovery(model, max_retries=5, engine=engine),
|
||||
timeout_seconds=0,
|
||||
hard_timeout=0,
|
||||
)
|
||||
assert engine.attempt["phase"] == "cancelled"
|
||||
assert engine.attempt["cancel_reason"] == "outer_timeout"
|
||||
assert engine.attempt["timeout_scope"] == "outer"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sdk_stop_preserves_cause_through_nested_request_wrappers(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
||||
engine = Engine()
|
||||
started = asyncio.Event()
|
||||
|
||||
async def model():
|
||||
started.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
control = task_control.TaskControl("task", object())
|
||||
agent = Agent.__new__(Agent)
|
||||
agent._current_task = asyncio.create_task(
|
||||
control.run_interruptibly(
|
||||
llm.invoke_llm_with_timeout_message(
|
||||
llm._run_with_recovery(model, max_retries=5, engine=engine), timeout_seconds=60
|
||||
)
|
||||
)
|
||||
)
|
||||
await asyncio.wait_for(started.wait(), 2)
|
||||
agent.stop_current_task()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await asyncio.wait_for(agent._current_task, 2)
|
||||
assert engine.attempt["phase"] == "cancelled"
|
||||
assert engine.attempt["cancel_reason"] == "user_stop"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_timer_is_identified_without_triggering_an_extra_request():
|
||||
engine = Engine()
|
||||
token = llm._FALLBACK_AVAILABLE.set(True)
|
||||
calls = 0
|
||||
|
||||
async def model():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
await asyncio.Event().wait()
|
||||
|
||||
try:
|
||||
with pytest.raises(LLMExhaustedError):
|
||||
await llm._run_with_recovery(model, request_timeout=0.01, max_retries=0, engine=engine)
|
||||
finally:
|
||||
llm._FALLBACK_AVAILABLE.reset(token)
|
||||
assert engine.attempt["phase"] == "failed"
|
||||
assert engine.attempt["timeout_scope"] == "request"
|
||||
assert "cancel_reason" not in engine.attempt
|
||||
assert calls == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unattributed_cancellation_does_not_infer_a_user_action_or_store_its_text():
|
||||
engine = Engine()
|
||||
started = asyncio.Event()
|
||||
|
||||
async def model():
|
||||
started.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
running = asyncio.create_task(llm._run_with_recovery(model, max_retries=5, engine=engine))
|
||||
await asyncio.wait_for(started.wait(), 2)
|
||||
running.cancel("private user guidance")
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await running
|
||||
assert engine.attempt["cancel_reason"] == "unknown"
|
||||
assert "private user guidance" not in repr(engine.records)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_http_499_preserves_server_id_but_does_not_infer_local_cancellation():
|
||||
engine = Engine()
|
||||
token = llm._FALLBACK_AVAILABLE.set(True)
|
||||
|
||||
class ServerError(Exception):
|
||||
status_code = 499
|
||||
request_id = "server-request-499"
|
||||
|
||||
async def model():
|
||||
raise ServerError("Request closed")
|
||||
|
||||
try:
|
||||
with pytest.raises(LLMExhaustedError):
|
||||
await llm._run_with_recovery(model, max_retries=0, engine=engine)
|
||||
finally:
|
||||
llm._FALLBACK_AVAILABLE.reset(token)
|
||||
assert engine.attempt["phase"] == "failed"
|
||||
assert engine.attempt["http_status"] == 499
|
||||
assert engine.attempt["server_request_id"] == "server-request-499"
|
||||
assert "cancel_reason" not in engine.attempt
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pause_cancellation_uses_the_existing_checkpoint(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
||||
engine = Engine()
|
||||
started = asyncio.Event()
|
||||
paused = asyncio.Event()
|
||||
events = []
|
||||
|
||||
class Lock:
|
||||
def release(self):
|
||||
events.append("released")
|
||||
|
||||
async def model():
|
||||
started.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
async def checkpoint(**_kwargs):
|
||||
events.append("checkpoint")
|
||||
paused.set()
|
||||
|
||||
control = task_control.TaskControl("task", Lock())
|
||||
monkeypatch.setattr(control, "checkpoint", checkpoint)
|
||||
running = asyncio.create_task(
|
||||
control.run_interruptibly(llm._run_with_recovery(model, max_retries=5, engine=engine))
|
||||
)
|
||||
await asyncio.wait_for(started.wait(), 2)
|
||||
task_control.request_pause("task")
|
||||
await asyncio.wait_for(paused.wait(), 2)
|
||||
assert await running == (True, None)
|
||||
assert engine.attempt["cancel_reason"] == "task_paused"
|
||||
assert events == ["checkpoint"]
|
||||
@@ -191,6 +191,8 @@ async def test_total_budget_expiry_is_not_user_cancellation_or_a_new_retry(
|
||||
stored = engine.latest("llm_diagnostic")
|
||||
assert stored["phase"] == "failed"
|
||||
assert stored["category"] == "final_review_budget_exhausted"
|
||||
assert stored["timeout_scope"] == "final_review"
|
||||
assert "cancel_reason" not in stored
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -77,7 +77,10 @@ def test_minimal_list_shows_hint_for_empty_inputs_and_errors():
|
||||
)
|
||||
text, items, labels = format_minimal_list_with_elements(elements[1:], 1000, 1000)
|
||||
lines = text.splitlines()
|
||||
assert lines[0] == "[1] Hint: 'Search settings' | Bounds: [0,0][100,100]"
|
||||
assert (
|
||||
lines[0]
|
||||
== "[1] Hint: 'Search settings' | Bounds: [0,0][100,100] | Editable: true | Value: empty"
|
||||
)
|
||||
assert lines[1] == "[2] Text: 'abc' | Bounds: [0,200][100,300] | Error: 'Password too short'"
|
||||
assert len(lines) == 2 and labels == ["1", "2"]
|
||||
assert items[0]["is_hint"] is True and items[0]["text"] == "Search settings"
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
"""A hint, OCR label and real editable value are different kinds of proof."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.drivers.android.hierarchy import parse_ui_hierarchy
|
||||
from artemis.utils.action_target import observation_anchor
|
||||
from artemis.utils.input_state import input_state, verify_input
|
||||
from artemis.utils.ui_filter import filter_ui_hierarchy
|
||||
from artemis.utils.visualization import format_minimal_list_with_elements
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("extra", "status"),
|
||||
[
|
||||
({"text": "example", "hint": "example", "showing-hint-text": "true"}, "empty"),
|
||||
({"text": "example", "hint": "example"}, "unknown"),
|
||||
({"text": "example", "hint": "example", "showing-hint-text": "false"}, "present"),
|
||||
({"text": "", "hint": "example"}, "empty"),
|
||||
({"hint": "example"}, "unknown"),
|
||||
({"text": "actual"}, "present"),
|
||||
],
|
||||
)
|
||||
def test_editable_value_evidence(extra, status):
|
||||
evidence = input_state({"class": "android.widget.EditText", **extra})
|
||||
assert evidence["input_value_state"] == status
|
||||
if status != "present":
|
||||
assert evidence["input_value"] in (None, "")
|
||||
|
||||
|
||||
def test_static_json_and_ocr_cannot_prove_input():
|
||||
static = {"class": "android.widget.TextView", "text": '{"pairing":"example"}'}
|
||||
assert input_state(static) == {}
|
||||
node = {
|
||||
"class": "android.widget.EditText",
|
||||
"text": "",
|
||||
"hint": "example",
|
||||
"bounds": "[0,0][100,100]",
|
||||
"ocr_elements": [{"text": "example", "bounds": [0, 0, 100, 100]}],
|
||||
}
|
||||
lines, indexed, _ = format_minimal_list_with_elements([node], 100, 100)
|
||||
assert len(indexed) == 1
|
||||
assert indexed[0]["input_value_state"] == "empty"
|
||||
assert "Hint:" in lines
|
||||
|
||||
|
||||
def test_filter_keeps_empty_input_and_does_not_promote_child_hint():
|
||||
node = {
|
||||
"class": "android.widget.EditText",
|
||||
"editable": True,
|
||||
"text": "",
|
||||
"bounds": "[0,0][100,100]",
|
||||
"children": [
|
||||
{"class": "android.widget.TextView", "text": "example", "bounds": "[0,0][100,100]"}
|
||||
],
|
||||
}
|
||||
filtered = filter_ui_hierarchy([node])
|
||||
assert filtered[0]["text"] == ""
|
||||
assert input_state(filtered[0])["input_value_state"] == "empty"
|
||||
|
||||
|
||||
def test_fresh_value_must_belong_to_the_actual_input():
|
||||
node = {
|
||||
"class": "android.widget.EditText",
|
||||
"resource-id": "pair",
|
||||
"text": "actual",
|
||||
"focused": True,
|
||||
}
|
||||
assert verify_input([node], {"target_resource_id": "pair"}) == "present"
|
||||
assert verify_input([node], {"target_resource_id": "other"}) == "unknown"
|
||||
assert verify_input([{**node, "password": True}]) == "unknown"
|
||||
|
||||
|
||||
def test_driver_parser_keeps_showing_hint_and_empty_editable_nodes():
|
||||
nodes = parse_ui_hierarchy(
|
||||
'<hierarchy><node class="android.widget.EditText" bounds="[0,0][100,100]" editable="true" text="example" hint="example" showing-hint-text="true" focused="true"/></hierarchy>'
|
||||
)
|
||||
assert len(nodes) == 1
|
||||
assert nodes[0]["input_value"] == ""
|
||||
assert nodes[0]["input_value_state"] == "empty"
|
||||
assert nodes[0]["focused"] is True
|
||||
|
||||
|
||||
def test_past_anchor_does_not_change_when_index_is_reused():
|
||||
state = SimpleNamespace(
|
||||
latest_screenshot="before.png", indexed_elements=[{"text": "Connect", "center": [1, 2]}]
|
||||
)
|
||||
old = observation_anchor(state)
|
||||
state.latest_screenshot = "after.png"
|
||||
state.indexed_elements = [{"text": "Keyboard key", "center": [3, 4]}]
|
||||
assert old["target_image_name"] == "before"
|
||||
assert observation_anchor(state)["target_snapshot_id"] != old["target_snapshot_id"]
|
||||
@@ -0,0 +1,276 @@
|
||||
"""Reject unfinished Responses output before parsers or tool consumers see it."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import httpx
|
||||
from langchain_core.messages import AIMessage, AIMessageChunk, HumanMessage, ToolMessage
|
||||
from langchain_core.runnables import RunnableLambda
|
||||
from pydantic import BaseModel, SecretStr
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from artemis.llm.reliability import FailureCategory, LLMExhaustedError
|
||||
from artemis.llm.response_guard import ResponseIntegrityError, ensure_response_complete
|
||||
from artemis.llm.router import ModelEndpoint, ModelFactory
|
||||
from artemis.services import llm as llm_service
|
||||
from artemis.services.llm import RobustChatModelWrapper, acomplete
|
||||
|
||||
|
||||
def response_body(status, arguments, name):
|
||||
return {
|
||||
"id": "resp_test",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": status,
|
||||
"model": "test-model",
|
||||
"output": [
|
||||
{
|
||||
"type": "function_call",
|
||||
"id": "fc_test",
|
||||
"call_id": "call_test",
|
||||
"name": name,
|
||||
"arguments": arguments,
|
||||
"status": "completed",
|
||||
}
|
||||
],
|
||||
"incomplete_details": {"reason": "max_output_tokens"} if status == "incomplete" else None,
|
||||
"usage": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13},
|
||||
}
|
||||
|
||||
|
||||
def response_stream(status, arguments, name):
|
||||
body = response_body(status or "in_progress", arguments, name)
|
||||
events = [
|
||||
{"type": "response.created", "response": {**body, "status": "in_progress", "output": []}},
|
||||
{
|
||||
"type": "response.output_item.added",
|
||||
"output_index": 0,
|
||||
"item": {**body["output"][0], "arguments": "", "status": "in_progress"},
|
||||
},
|
||||
{
|
||||
"type": "response.function_call_arguments.delta",
|
||||
"output_index": 0,
|
||||
"item_id": "fc_test",
|
||||
"delta": arguments,
|
||||
},
|
||||
]
|
||||
if status:
|
||||
events.append({"type": "response." + status, "response": body})
|
||||
return "".join("data: " + json.dumps(event) + "\n\n" for event in events) + "data: [DONE]\n\n"
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def transport_case(monkeypatch, tmp_path):
|
||||
from langchain_openai import ChatOpenAI
|
||||
import openai
|
||||
|
||||
requests, outputs, clients, events = [], [], [], []
|
||||
|
||||
def handle(request):
|
||||
payload = json.loads(request.content)
|
||||
requests.append(payload)
|
||||
status, arguments = outputs.pop(0)
|
||||
name = payload.get("tools", [{"name": "finish"}])[0]["name"]
|
||||
if payload.get("stream"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
text=response_stream(status, arguments, name),
|
||||
headers={"content-type": "text/event-stream"},
|
||||
)
|
||||
return httpx.Response(200, json=response_body(status, arguments, name))
|
||||
|
||||
def http_client(**kwargs):
|
||||
client = httpx.AsyncClient(transport=httpx.MockTransport(handle), **kwargs)
|
||||
clients.append(client)
|
||||
return client
|
||||
|
||||
monkeypatch.setattr(openai, "DefaultAsyncHttpxClient", http_client)
|
||||
monkeypatch.setattr(llm_service.settings, "OPENAI_API_KEY", SecretStr("test-secret"))
|
||||
monkeypatch.setattr(llm_service.settings, "OPENAI_BASE_URL", "https://responses.example/v1")
|
||||
monkeypatch.setattr(llm_service.settings, "OPENAI_MAX_RETRIES", 5)
|
||||
monkeypatch.setattr(llm_service.settings, "OPENAI_RETRY_INTERVAL_SECONDS", 0)
|
||||
monkeypatch.delenv("ARTEMIS_FAKE_LLM", raising=False)
|
||||
monkeypatch.setattr(llm_service, "PAUSE_FILE", tmp_path / ".artemis_paused")
|
||||
monkeypatch.setattr(llm_service, "_wait_for_resume", AsyncMock(return_value=False))
|
||||
monkeypatch.setattr(llm_service, "_get_current_data_engine", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
llm_service, "_record_llm_event", lambda *args, **kwargs: events.append(args)
|
||||
)
|
||||
llm_service._ENDPOINT_BREAKER._states.clear()
|
||||
llm_service._NON_STREAMING_ENDPOINTS.clear()
|
||||
endpoint = ModelEndpoint(model_name="test-model", max_retries=0)
|
||||
model = ModelFactory.create_model(endpoint)
|
||||
assert isinstance(model, ChatOpenAI)
|
||||
yield SimpleNamespace(
|
||||
model=model, endpoint=endpoint, requests=requests, outputs=outputs, events=events
|
||||
)
|
||||
for client in clients:
|
||||
await client.aclose()
|
||||
llm_service._ENDPOINT_BREAKER._states.clear()
|
||||
llm_service._NON_STREAMING_ENDPOINTS.clear()
|
||||
|
||||
|
||||
def wrapped(case):
|
||||
return RobustChatModelWrapper(case.model, endpoint=case.endpoint).bind_tools(
|
||||
[
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "finish",
|
||||
"description": "Record the verdict",
|
||||
"parameters": {"type": "object", "properties": {"passed": {"type": "boolean"}}},
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status", [None, "incomplete", "failed"])
|
||||
@pytest.mark.parametrize("arguments", ['{"passed":true}', '{"passed":true'])
|
||||
async def test_unfinished_stream_never_returns_a_verdict_or_tool_call(
|
||||
transport_case, status, arguments
|
||||
):
|
||||
case = transport_case
|
||||
case.outputs.append((status, arguments))
|
||||
with pytest.raises(LLMExhaustedError) as failure:
|
||||
await wrapped(case).complete([HumanMessage(content="Verify the task")])
|
||||
assert failure.value.failure.category is FailureCategory.INVALID_OUTPUT
|
||||
assert len(case.requests) == 1
|
||||
resets = [payload for event, payload in case.events if event == "llm_stream_reset"]
|
||||
assert len(resets) == 1 and resets[0]["action"] == "discard"
|
||||
assert not llm_service._NON_STREAMING_ENDPOINTS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("arguments", ['{"passed":true', ""])
|
||||
async def test_completed_event_cannot_repair_invalid_tool_json(transport_case, arguments):
|
||||
case = transport_case
|
||||
case.outputs.append(("completed", arguments))
|
||||
with pytest.raises(LLMExhaustedError) as failure:
|
||||
await wrapped(case).complete([])
|
||||
assert failure.value.failure.category is FailureCategory.INVALID_OUTPUT
|
||||
assert len(case.requests) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("arguments", [None, "[]", "null", '{"passed":true'])
|
||||
def test_original_function_call_arguments_must_be_a_complete_json_object(arguments):
|
||||
message = AIMessage(
|
||||
content=[{"type": "function_call", "arguments": arguments}],
|
||||
response_metadata={"model_provider": "openai", "status": "completed"},
|
||||
)
|
||||
with pytest.raises(ResponseIntegrityError, match="invalid_tool_arguments"):
|
||||
ensure_response_complete(message)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completed_stream_returns_original_valid_arguments(transport_case):
|
||||
case = transport_case
|
||||
case.outputs.append(("completed", '{"passed":true}'))
|
||||
result = await wrapped(case).complete([])
|
||||
assert result.response_metadata["status"] == "completed"
|
||||
assert result.tool_calls[0]["args"] == {"passed": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("recovers", [False, True])
|
||||
async def test_integrity_retry_is_bounded_and_keeps_existing_tool_evidence(
|
||||
transport_case, recovers
|
||||
):
|
||||
case = transport_case
|
||||
case.endpoint.max_retries = 5
|
||||
case.outputs.extend(
|
||||
[(None, '{"passed":false}'), ("completed" if recovers else None, '{"passed":true}')]
|
||||
)
|
||||
messages = [
|
||||
HumanMessage(content="Inspect then verify"),
|
||||
AIMessage(content="", tool_calls=[{"name": "inspect", "args": {}, "id": "prior_call"}]),
|
||||
ToolMessage(content="Existing screenshot evidence", tool_call_id="prior_call"),
|
||||
]
|
||||
if recovers:
|
||||
result = await wrapped(case).complete(messages)
|
||||
assert result.tool_calls[0]["args"] == {"passed": True}
|
||||
else:
|
||||
with pytest.raises(LLMExhaustedError):
|
||||
await wrapped(case).complete(messages)
|
||||
assert len(case.requests) == 2
|
||||
assert case.requests[0]["input"] == case.requests[1]["input"]
|
||||
assert any(
|
||||
item.get("call_id") == "prior_call" and item.get("output") == "Existing screenshot evidence"
|
||||
for item in case.requests[1]["input"]
|
||||
)
|
||||
assert isinstance(messages[-1], ToolMessage)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status", ["incomplete", "in_progress", "queued", "cancelled"])
|
||||
async def test_non_streaming_response_also_requires_completed(transport_case, status):
|
||||
case = transport_case
|
||||
case.outputs.append((status, '{"passed":true}'))
|
||||
with pytest.raises(LLMExhaustedError) as failure:
|
||||
await wrapped(case).ainvoke([])
|
||||
assert failure.value.failure.category is FailureCategory.INVALID_OUTPUT
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("include_raw", [False, True])
|
||||
async def test_structured_output_is_checked_before_returning_parsed_verdict(
|
||||
transport_case, include_raw
|
||||
):
|
||||
class Verdict(BaseModel):
|
||||
passed: bool
|
||||
|
||||
case = transport_case
|
||||
case.outputs.append(("incomplete", '{"passed":true}'))
|
||||
model = RobustChatModelWrapper(case.model, endpoint=case.endpoint)
|
||||
with pytest.raises(LLMExhaustedError):
|
||||
await model.with_structured_output(Verdict, include_raw=include_raw).ainvoke("Verify")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_downstream_parser_never_receives_incomplete_response(transport_case):
|
||||
case = transport_case
|
||||
case.outputs.append(("incomplete", '{"passed":true}'))
|
||||
parsed = []
|
||||
chain = wrapped(case) | RunnableLambda(lambda value: parsed.append(value))
|
||||
with pytest.raises(LLMExhaustedError):
|
||||
await chain.ainvoke([])
|
||||
assert parsed == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raw_model_compatibility_path_does_not_accept_missing_terminal_event(transport_case):
|
||||
case = transport_case
|
||||
case.outputs.append((None, '{"passed":true}'))
|
||||
with pytest.raises(ValueError):
|
||||
await acomplete(case.model, [])
|
||||
assert len(case.requests) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancellation_still_closes_stream_without_retry(transport_case):
|
||||
started, closed = asyncio.Event(), []
|
||||
|
||||
class StreamingModel:
|
||||
async def astream(self, *args, **kwargs):
|
||||
try:
|
||||
yield AIMessageChunk(
|
||||
content="partial", response_metadata={"model_provider": "openai"}
|
||||
)
|
||||
started.set()
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
closed.append(True)
|
||||
|
||||
task = asyncio.create_task(RobustChatModelWrapper(StreamingModel()).complete([]))
|
||||
await asyncio.wait_for(started.wait(), timeout=2)
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
assert closed == [True]
|
||||
assert not transport_case.requests
|
||||
resets = [payload for event, payload in transport_case.events if event == "llm_stream_reset"]
|
||||
assert len(resets) == 1 and resets[0]["reason"] == "incomplete_response"
|
||||
@@ -271,6 +271,63 @@ async def test_receipt_disk_failure_preserves_successful_write_result(monkeypatc
|
||||
assert outcome.side_effect != "none"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_popup_source_updates_cached_observation_target():
|
||||
runtime = PluginRuntime.__new__(PluginRuntime)
|
||||
runtime.ctx = None
|
||||
runtime.bindings = {
|
||||
"write": ("visual-test", Tool(name="write", inputSchema={"type": "object"}))
|
||||
}
|
||||
runtime.packages = {
|
||||
"visual-test": (
|
||||
manifest(
|
||||
capabilities={
|
||||
"kind": "visual",
|
||||
"tools": {
|
||||
"observe": {
|
||||
"effect": "read",
|
||||
"observation": True,
|
||||
"target_argument": "window_id",
|
||||
},
|
||||
"write": {"effect": "write", "target_argument": "window_id"},
|
||||
},
|
||||
}
|
||||
),
|
||||
SimpleNamespace(argument_constraints={}, secrets={}),
|
||||
None,
|
||||
)
|
||||
}
|
||||
runtime.sessions = {
|
||||
"visual-test": SimpleNamespace(
|
||||
call=AsyncMock(
|
||||
return_value=CallToolResult(
|
||||
content=[ImageContent(type="image", mimeType="image/png", data="aW1hZ2U=")],
|
||||
structuredContent={
|
||||
"artemis": {
|
||||
"version": 1,
|
||||
"execution": "completed",
|
||||
"source": {"kind": "computer", "target_id": "123"},
|
||||
}
|
||||
},
|
||||
)
|
||||
),
|
||||
stop_uncertain=False,
|
||||
)
|
||||
}
|
||||
runtime.ensure_resources_ready = lambda: None
|
||||
runtime._image_targets = {}
|
||||
runtime.last_images = {}
|
||||
runtime._observation_calls = {"visual-test": ("observe", {"pid": 7, "window_id": 10})}
|
||||
runtime._recordings = None
|
||||
first = await runtime.call("write", {"pid": 7, "window_id": 10})
|
||||
assert first.status == "success"
|
||||
assert first.pre_image_valid is False
|
||||
assert runtime._image_targets["visual-test"][0] == "123"
|
||||
assert runtime._observation_calls["visual-test"] == ("observe", {"pid": 7, "window_id": 123})
|
||||
second = await runtime.call("write", {"pid": 7, "window_id": 123})
|
||||
assert second.pre_image_valid is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_v2_observation_without_task_context_never_starts_recording(monkeypatch):
|
||||
runtime = PluginRuntime.__new__(PluginRuntime)
|
||||
|
||||
@@ -18,6 +18,7 @@ from artemis.llm.diagnostics import (
|
||||
)
|
||||
from artemis.llm.router import ModelEndpoint
|
||||
from artemis.services import llm
|
||||
from artemis.runtime.cancellation import CancellationReason
|
||||
|
||||
|
||||
class Engine:
|
||||
@@ -121,6 +122,10 @@ async def test_headers_without_body_timeout_then_retry_keeps_evidence(monkeypatc
|
||||
assert failed["usage"] is None and "first_output_at" not in failed
|
||||
assert final["request_id"] == failed["request_id"]
|
||||
assert final["attempt_id"] != failed["attempt_id"] and final["attempt"] == 2
|
||||
assert final["previous_attempt_id"] == failed["attempt_id"]
|
||||
retry = next(r["payload"] for r in engine.records if r["name"] == "llm_retry")
|
||||
assert retry["request_id"] == failed["request_id"]
|
||||
assert retry["attempt_id"] == failed["attempt_id"]
|
||||
assert final["phase"] == "completed" and final["usage"]["total_tokens"] == 12
|
||||
assert (
|
||||
final["started_at"] <= final["headers_at"] <= final["first_output_at"] <= final["ended_at"]
|
||||
@@ -226,6 +231,72 @@ async def test_concurrent_requests_do_not_mix_server_ids():
|
||||
assert second["server_request_id"] == "two"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_diagnostic_ids_survive_streaming_and_structured_completion_without_header_bodies():
|
||||
engine = Engine()
|
||||
attempt = RequestAttempt(
|
||||
engine, request_id="local", attempt=1, timeout=300, max_retries=1, metadata={}
|
||||
)
|
||||
token = ACTIVE_ATTEMPT.set(attempt)
|
||||
traceparent = "00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-01"
|
||||
try:
|
||||
request = httpx.Request(
|
||||
"POST", "https://example.test/responses", headers={"x-client-request-id": "caller-id"}
|
||||
)
|
||||
await observe_request_headers(request)
|
||||
await observe_response_headers(
|
||||
httpx.Response(
|
||||
200,
|
||||
headers={
|
||||
"x-request-id": "server-id",
|
||||
"x-trace-id": "gateway-id",
|
||||
"traceparent": traceparent,
|
||||
"set-cookie": "private cookie",
|
||||
"authorization": "private credential",
|
||||
},
|
||||
)
|
||||
)
|
||||
attempt.chunk(AIMessageChunk(content="hello", response_metadata={"id": "resp-first"}))
|
||||
attempt.finish(
|
||||
"completed",
|
||||
response={
|
||||
"raw": AIMessageChunk(
|
||||
content="hello", response_metadata={"id": "resp-final", "status": "completed"}
|
||||
),
|
||||
"parsed": {"private": "business details"},
|
||||
},
|
||||
)
|
||||
finally:
|
||||
ACTIVE_ATTEMPT.reset(token)
|
||||
assert request.headers["x-client-request-id"] == "caller-id"
|
||||
assert attempt.data["client_request_id"] == "caller-id"
|
||||
assert attempt.data["server_request_id"] == "server-id"
|
||||
assert attempt.data["server_trace_id"] == "gateway-id"
|
||||
assert attempt.data["traceparent"] == traceparent
|
||||
assert attempt.data["response_id"] == "resp-final"
|
||||
assert attempt.data["response_status"] == "completed"
|
||||
assert "private" not in json.dumps(engine.records, default=str)
|
||||
|
||||
|
||||
def test_invalid_or_overlong_correlation_headers_are_omitted():
|
||||
attempt = RequestAttempt(
|
||||
Engine(), request_id="local", attempt=1, timeout=300, max_retries=1, metadata={}
|
||||
)
|
||||
attempt.headers(
|
||||
httpx.Response(
|
||||
200,
|
||||
headers={
|
||||
"x-request-id": "x" * 257,
|
||||
"x-trace-id": "not an id",
|
||||
"traceparent": "not a trace",
|
||||
},
|
||||
)
|
||||
)
|
||||
assert attempt.data["server_request_id"] is None
|
||||
assert attempt.data["server_trace_id"] is None
|
||||
assert "traceparent" not in attempt.data
|
||||
|
||||
|
||||
def test_error_redaction_and_image_headers():
|
||||
import base64
|
||||
import io
|
||||
@@ -372,11 +443,14 @@ async def test_cancelling_retry_wait_does_not_leave_a_live_request(monkeypatch):
|
||||
|
||||
task = asyncio.create_task(llm._run_with_recovery(timeout, max_retries=1))
|
||||
await asyncio.wait_for(waiting.wait(), 2)
|
||||
task.cancel()
|
||||
task.cancel(CancellationReason.GUIDANCE_REPLACED)
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
assert engine.records[-1]["payload"]["phase"] == "cancelled"
|
||||
assert "模型请求" in engine.records[-1]["payload"]["error"]
|
||||
assert engine.records[-1]["payload"]["cancel_reason"] == "guidance_replaced"
|
||||
assert engine.records[-1]["payload"]["cancel_phase"] == "retry_wait"
|
||||
assert "timeout_scope" not in engine.records[-1]["payload"]
|
||||
|
||||
|
||||
def test_diagnostic_api_limits_preview_but_downloads_complete_history(tmp_path, monkeypatch):
|
||||
|
||||
@@ -361,7 +361,7 @@ async def test_launch_app_failure_when_top_task_is_another_app(
|
||||
assert success is False
|
||||
assert f"Failed to launch {SETTINGS}" in error_msg
|
||||
assert mock_controller.launch_app.call_count == 2
|
||||
assert mock_controller.terminate_app.call_count == 1
|
||||
mock_controller.terminate_app.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -399,12 +399,13 @@ async def test_launch_timeout_message_names_foreground_package_and_task(
|
||||
@patch("artemis.utils.app_launch_utils.get_foreground_task")
|
||||
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
|
||||
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
|
||||
async def test_launch_app_null_focus_keeps_polling_without_reading_task(
|
||||
async def test_launch_app_null_focus_without_task_evidence_keeps_polling(
|
||||
mock_get_foreground, mock_controller_cls, mock_get_task, mock_context
|
||||
):
|
||||
"""mCurrentFocus=null is a loading state: no task lookup, then success once focused."""
|
||||
"""Unknown focus and task state remain loading until a focused window appears."""
|
||||
_controller(mock_controller_cls)
|
||||
mock_get_foreground.side_effect = [None, None, YOUTUBE]
|
||||
mock_get_task.return_value = None
|
||||
|
||||
success, error_msg = await launch_app_with_retries(
|
||||
mock_context, YOUTUBE, max_retries=1, max_poll_seconds=3
|
||||
@@ -412,7 +413,67 @@ async def test_launch_app_null_focus_keeps_polling_without_reading_task(
|
||||
|
||||
assert success is True
|
||||
assert error_msg is None
|
||||
mock_get_task.assert_not_called()
|
||||
assert mock_get_task.call_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"dump, ready",
|
||||
[
|
||||
(DUMP_SETTINGS_TASK_WITH_SEARCH_ON_TOP, True),
|
||||
(DUMP_NO_RESUMED, False),
|
||||
(DUMP_HOME, False),
|
||||
("", False),
|
||||
],
|
||||
ids=["resumed-app", "visible-only", "other-app", "unknown"],
|
||||
)
|
||||
async def test_null_focus_requires_resumed_foreground_task_evidence(mock_context, dump, ready):
|
||||
with (
|
||||
patch("artemis.utils.app_launch_utils.UnifiedMobileController") as controller_cls,
|
||||
patch(
|
||||
"artemis.utils.app_launch_utils.get_current_foreground_package_async",
|
||||
new_callable=AsyncMock,
|
||||
return_value=None,
|
||||
),
|
||||
patch(
|
||||
"artemis.utils.app_launch_utils.get_foreground_task",
|
||||
return_value=parse_foreground_task(dump),
|
||||
),
|
||||
patch("artemis.utils.app_launch_utils.asyncio.sleep", new_callable=AsyncMock),
|
||||
):
|
||||
controller = _controller(controller_cls)
|
||||
success, detail = await launch_app_with_retries(
|
||||
mock_context, SETTINGS, max_retries=2, max_poll_seconds=1
|
||||
)
|
||||
assert success is ready
|
||||
if ready:
|
||||
assert detail is None
|
||||
controller.launch_app.assert_awaited_once_with(SETTINGS)
|
||||
else:
|
||||
assert "Current foreground: None" in detail
|
||||
assert controller.launch_app.await_count == 2
|
||||
controller.terminate_app.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initial_launch_skips_relaunch_with_null_focus_and_resumed_app_task(mock_context):
|
||||
with (
|
||||
patch(
|
||||
"artemis.utils.app_launch_utils.get_current_foreground_package_async",
|
||||
new_callable=AsyncMock,
|
||||
return_value=None,
|
||||
),
|
||||
patch(
|
||||
"artemis.utils.app_launch_utils.get_foreground_task",
|
||||
return_value=parse_foreground_task(DUMP_SETTINGS_TASK_WITH_SEARCH_ON_TOP),
|
||||
),
|
||||
patch(
|
||||
"artemis.utils.app_launch_utils.launch_app_with_retries", new_callable=AsyncMock
|
||||
) as launch,
|
||||
):
|
||||
result = await _handle_initial_app_launch(mock_context, SETTINGS)
|
||||
assert result.locked_app_initial_launch_success is True
|
||||
launch.assert_not_awaited()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -467,6 +528,7 @@ async def test_transient_launch_failure_is_bounded_and_keeps_cause(mock_context)
|
||||
assert result is False
|
||||
assert detail == "device offline"
|
||||
assert controller.launch_app.await_count == 3
|
||||
controller.terminate_app.assert_not_called()
|
||||
poll.assert_not_called()
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user