feat: release 2.4.0 with original cases and automatic human review
This commit is contained in:
1 parent
baf069854b
commit
4dc2057d43
66 files changed
+1390
-206
No files matched your search
@@ -908,6 +908,85 @@ def test_reports_do_not_execute_formulas_or_invent_pass(isolated):
|
||||
assert "未验证".encode() in doc.read("word/document.xml")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("result_status", ["passed", "failed", "blocked", "inconclusive"])
|
||||
def test_report_without_output_body_uses_effective_verdict_and_evidence(isolated, result_status):
|
||||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||||
isolated.save_job({**job, "status": "completed"})
|
||||
folder = reports.TRACES_PATH / job["session_id"]
|
||||
folder.mkdir(parents=True)
|
||||
(folder / "run_outcome.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"task_status": "completed",
|
||||
"test_result": {
|
||||
"status": result_status,
|
||||
"reason": "实际验收依据",
|
||||
"evidence": ["记录中的证据"],
|
||||
},
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
report = reports.session_report(job["session_id"])
|
||||
assert report["verdict"] in report["content"]
|
||||
assert "实际验收依据" in report["content"]
|
||||
assert "记录中的证据" in report["content"]
|
||||
assert "未保存独立输出正文" in report["content"]
|
||||
assert "没有保存最终结论" not in report["content"]
|
||||
assert report["test_result"]["status"] == result_status
|
||||
|
||||
|
||||
def test_report_without_output_body_keeps_human_review_authoritative(isolated):
|
||||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||||
isolated.save_job({**job, "status": "completed"})
|
||||
folder = reports.TRACES_PATH / job["session_id"]
|
||||
folder.mkdir(parents=True)
|
||||
(folder / "run_outcome.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"test_result": {"status": "passed", "reason": "AI 原判通过"},
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
isolated.save_case_review(job["session_id"], "failed", "用户确认实际结果不符合预期")
|
||||
report = reports.session_report(job["session_id"])
|
||||
assert "人工确认" in report["content"]
|
||||
assert "用户确认实际结果不符合预期" in report["content"]
|
||||
assert "AI 原判通过" not in report["content"]
|
||||
assert report["test_result"]["status"] == "failed"
|
||||
|
||||
|
||||
def test_report_projects_stored_android_receipts_without_rewriting_history(isolated):
|
||||
from tests.unit.data_engine.test_evidence_sources import android_receipt
|
||||
|
||||
sid = isolated.jobs(service.create_batch(payload())["id"])[0]["session_id"]
|
||||
receipt = android_receipt()["payload"]["result"]
|
||||
raw_payload = json.dumps({"args": {"action": "screenshot"}, "result": json.dumps(receipt)})
|
||||
with sqlite3.connect(isolated.path) as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO sessions(session_id,initial_goal,start_time,status) VALUES (?,?,?,?)",
|
||||
(sid, "跨端截图", 1, "completed"),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO steps(step_id,session_id,step_number) VALUES (?,?,?)", ("s", sid, 1)
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO traces(trace_id,session_id,step_id,type,name,timestamp,status,payload) VALUES (?,?,?,?,?,?,?,?)",
|
||||
("capture", sid, "s", "tool", "android_file", 2, "success", raw_payload),
|
||||
)
|
||||
report = reports.session_report(sid)
|
||||
source = report["evidence_sources"]["sources"][0]
|
||||
assert source["kind"] == "android"
|
||||
assert source["references"][0]["trace_id"] == "capture"
|
||||
assert report["evidence_sources"]["unattributed_step_count"] == 0
|
||||
with sqlite3.connect(isolated.path) as conn:
|
||||
assert (
|
||||
conn.execute("SELECT payload FROM traces WHERE trace_id='capture'").fetchone()[0]
|
||||
== raw_payload
|
||||
)
|
||||
|
||||
|
||||
def test_failed_report_exposes_logged_reason_and_formats_chinese_markdown(isolated):
|
||||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||||
isolated.save_job({**job, "status": "failed"})
|
||||
|
||||
@@ -110,6 +110,8 @@ def test_strip_trace_status_suffix_handles_every_terminal_marker():
|
||||
assert MediaService.strip_trace_status_suffix("web_1_PASS_2026") == "web_1"
|
||||
assert MediaService.strip_trace_status_suffix("web_1_FAIL_2026") == "web_1"
|
||||
assert MediaService.strip_trace_status_suffix("web_1_TESTFAIL_2026") == "web_1"
|
||||
assert MediaService.strip_trace_status_suffix("web_1_BLOCKED_2026") == "web_1"
|
||||
assert MediaService.strip_trace_status_suffix("web_1_UNVERIFIED_2026") == "web_1"
|
||||
assert MediaService.strip_trace_status_suffix("web_1") == "web_1"
|
||||
|
||||
|
||||
|
||||
@@ -44,6 +44,10 @@ async def test_list_sessions_uses_batched_history_metadata(monkeypatch):
|
||||
repo.get_llm_traces_for_profiles_map.return_value = {
|
||||
"session-3": ['{"model": "gemini-3.7-pro"}']
|
||||
}
|
||||
repo.get_recorded_model_info_map.return_value = {
|
||||
"session-1": {"id": "gpt-6.1-terra", "provider": "openai"},
|
||||
"session-2": {"id": "gpt-6-luna", "provider": "openai"},
|
||||
}
|
||||
repo.get_agent_trace_names_map.return_value = {}
|
||||
|
||||
monkeypatch.setattr(sessions_router, "session_repo", repo, raising=False)
|
||||
@@ -57,6 +61,10 @@ async def test_list_sessions_uses_batched_history_metadata(monkeypatch):
|
||||
result = await sessions_router.list_sessions()
|
||||
|
||||
assert len(result) == 3
|
||||
assert [row["model_info"]["id"] for row in result] == ["gpt-6.1-terra", "gpt-6-luna", ""]
|
||||
repo.get_recorded_model_info_map.assert_called_once_with(
|
||||
["session-1", "session-2", "session-3"]
|
||||
)
|
||||
repo.get_latest_video_recordings_map.assert_called_once_with()
|
||||
repo.get_llm_traces_for_profiles_map.assert_called_once_with(["session-3"])
|
||||
repo.get_agent_trace_names_map.assert_called_once_with()
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import json
|
||||
from unittest.mock import patch
|
||||
|
||||
from apps.admin_console.database.connection import db_session
|
||||
@@ -37,3 +38,31 @@ def test_session_list_harvest_rechecks_worker_liveness(tmp_path):
|
||||
assert repository.harvest_orphaned_sessions(["external-live"]) == 0
|
||||
|
||||
assert repository.get_session_status("external-live") == "running"
|
||||
|
||||
|
||||
def test_recorded_model_metadata_is_per_session_and_prefers_operator(tmp_path):
|
||||
db_path = tmp_path / "sessions.db"
|
||||
repository = SessionRepository(db_path)
|
||||
for session_id in ("one", "two", "no-record"):
|
||||
_insert_running_session(db_path, session_id, 1)
|
||||
with db_session(db_path) as conn:
|
||||
for trace_id, session_id, timestamp, payload in (
|
||||
("invalid", "one", 0, "{"),
|
||||
("planner", "one", 1, json.dumps({"role": "planner", "model": "other-model"})),
|
||||
(
|
||||
"operator",
|
||||
"one",
|
||||
2,
|
||||
json.dumps({"role": "operator", "model": "gpt-6.1-terra", "provider": "openai"}),
|
||||
),
|
||||
("flash", "two", 1, json.dumps({"role": "FlashRunner", "model": "gpt-6-luna"})),
|
||||
):
|
||||
conn.execute(
|
||||
"INSERT INTO traces(trace_id,session_id,type,timestamp,payload) VALUES (?,?,?,?,?)",
|
||||
(trace_id, session_id, "llm_diagnostic", timestamp, payload),
|
||||
)
|
||||
conn.commit()
|
||||
assert repository.get_recorded_model_info_map(["one", "two", "no-record"]) == {
|
||||
"one": {"id": "gpt-6.1-terra", "provider": "openai"},
|
||||
"two": {"id": "gpt-6-luna", "provider": ""},
|
||||
}
|
||||
@@ -99,6 +99,34 @@ def test_usage_flash_runner_counts_as_operator_and_has_no_tuning(tmp_path):
|
||||
assert usage["run_tuning"] is None
|
||||
|
||||
|
||||
def test_usage_uses_its_recorded_model_and_supports_legacy_endpoint_keys(tmp_path):
|
||||
db_path = tmp_path / "sessions.db"
|
||||
repo = SessionRepository(db_path)
|
||||
_insert_session(db_path, "models", {"profile": "flash"})
|
||||
_insert_usage(db_path, "models", "new", 1.0, {**_usage(100, 10), "model": "terra"})
|
||||
_insert_usage(
|
||||
db_path,
|
||||
"models",
|
||||
"legacy",
|
||||
2.0,
|
||||
{**_usage(200, 20), "source": "openai:terra:https://example.test:high"},
|
||||
)
|
||||
_insert_usage(
|
||||
db_path, "models", "lens", 3.0, {**_usage(300, 30), "source": "lens:step_capsule:terra"}
|
||||
)
|
||||
_insert_usage(db_path, "models", "unknown", 4.0, {**_usage(300, 30), "source": "other"})
|
||||
_insert_usage(
|
||||
db_path,
|
||||
"models",
|
||||
"fallback",
|
||||
5.0,
|
||||
{**_usage(300, 30), "source": "lens:step_capsule:fallback"},
|
||||
)
|
||||
models = {item["model"]: item for item in repo.get_session_usage("models")["by_node_model"]}
|
||||
assert models["terra"]["llm_calls"] == 3
|
||||
assert models["unknown"]["llm_calls"] == 2
|
||||
|
||||
|
||||
def test_usage_for_unknown_or_empty_session_is_zeroed(tmp_path):
|
||||
db_path = tmp_path / "sessions.db"
|
||||
repo = SessionRepository(db_path)
|
||||
@@ -112,6 +140,26 @@ def test_usage_for_unknown_or_empty_session_is_zeroed(tmp_path):
|
||||
assert usage["run_tuning"] is None
|
||||
|
||||
|
||||
def test_legacy_usage_preserves_model_colons_and_keeps_models_separate(tmp_path):
|
||||
db_path = tmp_path / "sessions.db"
|
||||
repo = SessionRepository(db_path)
|
||||
_insert_session(db_path, "models", {"profile": "flash"})
|
||||
sources = {
|
||||
"team/model:stable": "openai:team/model:stable:https://example.test:8443/v1:high",
|
||||
"team/model:preview": "openai:team/model:preview:http://[::1]:8080/v1:high",
|
||||
"unknown": "openai:team/model:preview:unparseable:high",
|
||||
}
|
||||
for index, source in enumerate(sources.values()):
|
||||
_insert_usage(
|
||||
db_path, "models", str(index), float(index), {**_usage(100, 10), "source": source}
|
||||
)
|
||||
|
||||
groups = {item["model"]: item for item in repo.get_session_usage("models")["by_node_model"]}
|
||||
assert groups.keys() == sources.keys()
|
||||
assert all(item["llm_calls"] == 1 for item in groups.values())
|
||||
assert all(item["prompt_tokens"] == 100 for item in groups.values())
|
||||
|
||||
|
||||
def test_usage_route_is_registered():
|
||||
from apps.admin_console.routers import sessions
|
||||
|
||||
|
||||
@@ -3,6 +3,30 @@ import json
|
||||
from apps.admin_console.database.repositories.step_repository import StepRepository
|
||||
|
||||
|
||||
def test_android_capture_receipt_survives_cleaning_without_becoming_image_store_id():
|
||||
receipt = {
|
||||
"ok": True,
|
||||
"action": "screenshot",
|
||||
"verified": True,
|
||||
"device_serial": "device-under-test",
|
||||
"trace_path": "artifacts/android/screen.png",
|
||||
"sha256": "a" * 64,
|
||||
"captured_at": "2026-10-28T00:00:00Z",
|
||||
"content": "large file content must not be copied into the task stream",
|
||||
}
|
||||
cleaned = StepRepository()._clean_tool_payload(
|
||||
{"args": {"action": "screenshot"}, "result": json.dumps(receipt)},
|
||||
"tool",
|
||||
"android_file",
|
||||
)
|
||||
assert cleaned["result"] == {key: value for key, value in receipt.items() if key != "content"}
|
||||
assert "post_image_name" not in cleaned
|
||||
wrapped = StepRepository()._clean_tool_payload(
|
||||
{"result": json.dumps({"outcome": json.dumps(receipt)})}, "tool", "android_file"
|
||||
)
|
||||
assert wrapped["result"] == cleaned["result"]
|
||||
|
||||
|
||||
def test_legacy_pause_log_is_normalized_to_failed_llm_trace():
|
||||
repository = StepRepository()
|
||||
trace = {
|
||||
|
||||
@@ -57,10 +57,13 @@ def manifest_payload():
|
||||
}
|
||||
|
||||
|
||||
def install(plugins):
|
||||
def install(plugins, original_case=False):
|
||||
package = BytesIO()
|
||||
manifest = manifest_payload()
|
||||
if original_case:
|
||||
manifest["workspace"]["panels"][0]["kind"] = "original_case"
|
||||
with zipfile.ZipFile(package, "w") as archive:
|
||||
archive.writestr("plugin.json", json.dumps(manifest_payload()))
|
||||
archive.writestr("plugin.json", json.dumps(manifest))
|
||||
install_archive(plugins, package.getvalue())
|
||||
plugins.set_enabled("review", True)
|
||||
|
||||
@@ -105,6 +108,86 @@ def save(client, values, revision=0, request_id=None):
|
||||
)
|
||||
|
||||
|
||||
def test_original_case_import_requires_enabled_plugin_and_keeps_execution_prompt(context):
|
||||
from apps.admin_console.services import task_import
|
||||
|
||||
store, plugins, client = context
|
||||
legacy = load_workbook(BytesIO(task_import.template()))
|
||||
assert legacy["任务"].max_column == 7
|
||||
legacy["任务"].append(["OLD", "", "", "旧用例", "", "原样执行", ""])
|
||||
legacy_bytes = BytesIO()
|
||||
legacy.save(legacy_bytes)
|
||||
assert "original_case" not in task_import.parse_workbook(legacy_bytes.getvalue())["tasks"][0]
|
||||
|
||||
install(plugins, original_case=True)
|
||||
workbook = load_workbook(BytesIO(task_import.template()))
|
||||
assert [cell.value for cell in workbook["任务"][1]] == [
|
||||
*task_import.HEADERS,
|
||||
*task_import.CASE_HEADERS,
|
||||
]
|
||||
workbook["任务"].append(
|
||||
[
|
||||
"CASE-001",
|
||||
"检测项目",
|
||||
"测试子项",
|
||||
"测试标题",
|
||||
"提取的约束",
|
||||
"只执行此提示词",
|
||||
"{}",
|
||||
"原始预置条件",
|
||||
"1. 第一步\n2. 第二步",
|
||||
"原始预期结果",
|
||||
]
|
||||
)
|
||||
stream = BytesIO()
|
||||
workbook.save(stream)
|
||||
case = task_import.parse_workbook(stream.getvalue())["tasks"][0]
|
||||
assert task_import.case_goal(case) == "只执行此提示词"
|
||||
assert case["preconditions"] == "提取的约束"
|
||||
assert case["original_case"] == {
|
||||
"preconditions": "原始预置条件",
|
||||
"steps": "1. 第一步\n2. 第二步",
|
||||
"expected": "原始预期结果",
|
||||
}
|
||||
assert task_import.parse_workbook(legacy_bytes.getvalue())["valid"]
|
||||
store.save_job({"session_id": "task-one", "status": "completed", "case": case})
|
||||
result = client.get("/api/task-extensions/task-one").json()
|
||||
assert "CASE-001" in result["case_title"] and "测试标题" in result["case_title"]
|
||||
assert result["original_case"] == case["original_case"]
|
||||
assert result["initial_prompt"] == "只执行此提示词"
|
||||
assert report_export.session_report("task-one")["original_case"] == case["original_case"]
|
||||
|
||||
plugins.set_enabled("review", False)
|
||||
assert load_workbook(BytesIO(task_import.template()))["任务"].max_column == 7
|
||||
with pytest.raises(ValueError, match="列名或顺序"):
|
||||
task_import.parse_workbook(stream.getvalue())
|
||||
|
||||
|
||||
def test_review_navigation_uses_batch_order_and_excludes_unfinished_jobs(context):
|
||||
store, _, client = context
|
||||
store.create_batch({"id": "review-batch", "status": "completed"}, [], {})
|
||||
for position, status in [(3, "completed"), (1, "failed"), (2, "pending")]:
|
||||
store.save_job(
|
||||
{
|
||||
"session_id": f"batch-task-{position}",
|
||||
"batch_id": "review-batch",
|
||||
"position": position,
|
||||
"status": status,
|
||||
"case": {"name": f"用例{position}", "goal": "测试"},
|
||||
}
|
||||
)
|
||||
first = client.get("/api/task-extensions/batch-task-1").json()["navigation"]
|
||||
assert first == {
|
||||
"scope": "batch",
|
||||
"position": 1,
|
||||
"total": 2,
|
||||
"previous": None,
|
||||
"next": "batch-task-3",
|
||||
}
|
||||
last = client.get("/api/task-extensions/batch-task-3").json()["navigation"]
|
||||
assert last["previous"] == "batch-task-1" and last["next"] is None
|
||||
|
||||
|
||||
def test_workspace_requires_v3_and_valid_bounded_declarations():
|
||||
for change in (
|
||||
{"schema_version": 2},
|
||||
|
||||
@@ -601,6 +601,45 @@ async def test_status_reports_external_global_owner_without_ipc_connection():
|
||||
assert result["goal"] == "CLI task: inspect settings"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("recorded", [True, False])
|
||||
async def test_status_uses_the_active_sessions_model(recorded):
|
||||
owner = DeviceLockOwner(
|
||||
pid=24680,
|
||||
process_created_at=1234.5,
|
||||
token="active-owner-token",
|
||||
device_id="emulator-5554",
|
||||
description="active task",
|
||||
acquired_at="2026-08-24T00:00:00+00:00",
|
||||
session_id="active",
|
||||
ingress="web",
|
||||
)
|
||||
state.queue_items.append(
|
||||
{"session_id": "active", "status": "running", "runtime_overrides": {"model": "terra"}}
|
||||
)
|
||||
with (
|
||||
patch.object(TaskQueueService, "ensure_worker_running"),
|
||||
patch(
|
||||
"apps.admin_console.routers.tasks.DeviceExecutionLock.get_active_owner",
|
||||
return_value=owner,
|
||||
),
|
||||
patch(
|
||||
"apps.admin_console.routers.tasks.DeviceExecutionLock.get_active_owners",
|
||||
return_value={},
|
||||
),
|
||||
patch("apps.admin_console.routers.tasks.session_repo") as repo,
|
||||
patch("apps.admin_console.routers.tasks.model_service") as models,
|
||||
):
|
||||
repo.get_latest_session.return_value = None
|
||||
repo.get_session_by_id.return_value = None
|
||||
repo.get_recorded_model_info_map.return_value = (
|
||||
{"active": {"id": "terra-recorded", "provider": "openai"}} if recorded else {}
|
||||
)
|
||||
models.get_active_model_info.return_value = {"id": "global", "provider": "openai"}
|
||||
result = await get_status()
|
||||
assert result["model_info"]["id"] == ("terra-recorded" if recorded else "terra")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancel_task_triggers_next_pending_task():
|
||||
executed_goals = []
|
||||
|
||||
@@ -379,9 +379,13 @@ def test_flash_records_the_executors_target_semantics(mock_context):
|
||||
runner,
|
||||
"click",
|
||||
click_args,
|
||||
metadata={"target_semantics": {"target_description": "play button"}},
|
||||
metadata={
|
||||
"target_semantics": {"target_description": "play button"},
|
||||
"source": {"kind": "android", "target_id": "selected-phone", "label": "安卓屏幕"},
|
||||
},
|
||||
)
|
||||
assert click["target_description"] == "play button"
|
||||
assert click["source"]["target_id"] == "selected-phone"
|
||||
assert "target_text" not in click
|
||||
assert format_actions_clean(click) == "Tapped 'play button' (self-described) at [320, 399]"
|
||||
|
||||
|
||||
@@ -124,6 +124,11 @@ async def test_planner_knows_computer_plugins_with_mobile_primary_screen(mock_co
|
||||
content = llm.astream.call_args.args[0][1].content
|
||||
text = "\n".join(part["text"] for part in content if part["type"] == "text")
|
||||
assert text.count("电脑插件:cua-driver") == 1
|
||||
assert "规划器只使用本阶段绑定的笔记工具" in text
|
||||
assert "不表示执行器缺少这些能力" in text
|
||||
assert "实际加载与操作回执" in text
|
||||
bound_names = {tool.name for tool in llm.bind_tools.call_args.args[0]}
|
||||
assert "plugin_load" not in bound_names
|
||||
assert f"当前主设备截图({'电脑' if target == 'computer' else '安卓'})" in text
|
||||
assert "不决定其他步骤的执行平台" in text
|
||||
|
||||
|
||||
@@ -122,6 +122,7 @@ def mock_mcp():
|
||||
async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that ValidatorNode executes actions successfully."""
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
|
||||
mock_context.device.device_id = "selected-phone"
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
state.initial_goal = "在真实审批表单上 Approve once,不要普通聊天代答或重复提交。"
|
||||
|
||||
@@ -142,6 +143,11 @@ async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
|
||||
assert len(report["execution"]) == 1
|
||||
assert report["execution"][0]["action"] == "tap"
|
||||
assert "attempts" not in report["execution"][0]
|
||||
assert report["execution"][0]["source"] == {
|
||||
"kind": "android",
|
||||
"target_id": "selected-phone",
|
||||
"label": "安卓屏幕",
|
||||
}
|
||||
# The Operator's tap verb was translated to the canonical click tool.
|
||||
assert len(mock_mcp.calls_for("click")) == 1
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
from copy import deepcopy
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.data_engine.evidence_sources import recorded_sources
|
||||
|
||||
|
||||
@@ -59,6 +61,21 @@ def test_sources_remain_separate_and_references_use_host_ids():
|
||||
assert value["sources"][1]["category"] == "interface"
|
||||
|
||||
|
||||
def test_window_label_change_does_not_attribute_all_steps_to_the_first_title():
|
||||
first = record("computer", "window", call_id="c1")
|
||||
second = record("computer", "window", call_id="c2")
|
||||
first["source"]["label"] = "first page"
|
||||
second["source"]["label"] = "second page"
|
||||
result = recorded_sources(
|
||||
[{"step_id": "s1", "action_taken": first}, {"step_id": "s2", "action_taken": second}]
|
||||
)
|
||||
source = result["sources"][0]
|
||||
assert result["total_sources"] == 1
|
||||
assert source["reference_count"] == 2
|
||||
assert source["label"] == ""
|
||||
assert source["label_varies"] is True
|
||||
|
||||
|
||||
def test_model_arguments_and_unsourced_images_do_not_fabricate_source_metadata():
|
||||
value = recorded_sources(
|
||||
[
|
||||
@@ -103,3 +120,98 @@ def test_partial_source_metadata_does_not_hide_unknown_actions_in_the_same_step(
|
||||
)
|
||||
assert value["total_sources"] == 1
|
||||
assert value["unattributed_step_count"] == 1
|
||||
|
||||
|
||||
def android_receipt():
|
||||
return {
|
||||
"trace_id": "capture-trace",
|
||||
"type": "tool",
|
||||
"name": "android_file",
|
||||
"status": "success",
|
||||
"payload": {
|
||||
"result": {
|
||||
"ok": True,
|
||||
"action": "screenshot",
|
||||
"verified": True,
|
||||
"device_serial": "test-phone",
|
||||
"trace_path": "artifacts/android/home.png",
|
||||
"sha256": "a" * 64,
|
||||
"captured_at": "2026-10-28T00:00:00Z",
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_android_receipts_join_computer_sources_and_deduplicate_nested_tool_traces():
|
||||
tool = android_receipt()
|
||||
duplicate = {**deepcopy(tool), "trace_id": "outer-wrapper"}
|
||||
steps = [
|
||||
{
|
||||
"step_id": "mixed",
|
||||
"action_taken": record("computer", "window"),
|
||||
"generic_tools": [tool, duplicate],
|
||||
},
|
||||
{"step_id": "final", "generic_tools": [tool]},
|
||||
{"step_id": "empty", "action_taken": None, "generic_tools": []},
|
||||
]
|
||||
before = deepcopy(steps)
|
||||
result = recorded_sources(steps)
|
||||
sources = {source["kind"]: source for source in result["sources"]}
|
||||
assert set(sources) == {"android", "computer"}
|
||||
assert sources["android"]["target_id"] == "test-phone"
|
||||
assert sources["android"]["reference_count"] == 2
|
||||
assert sources["android"]["references"][0]["trace_id"] == "capture-trace"
|
||||
assert sources["android"]["references"][0]["artifact_path"] == "artifacts/android/home.png"
|
||||
assert result["unattributed_step_count"] == 0
|
||||
assert steps == before
|
||||
|
||||
|
||||
@pytest.mark.parametrize("invalid", ["failed", "unverified", "different_tool", "arguments_only"])
|
||||
def test_only_successful_android_capture_receipts_supply_provenance(invalid):
|
||||
tool = android_receipt()
|
||||
if invalid == "failed":
|
||||
tool["status"] = "failed"
|
||||
elif invalid == "unverified":
|
||||
tool["payload"]["result"]["verified"] = False
|
||||
elif invalid == "different_tool":
|
||||
tool["name"] = "read_note"
|
||||
else:
|
||||
tool["payload"] = {"args": tool["payload"]["result"]}
|
||||
value = recorded_sources(
|
||||
[{"step_id": "s", "action_taken": {"action": "tap"}, "generic_tools": [tool]}]
|
||||
)
|
||||
assert value["sources"] == []
|
||||
assert value["unattributed_step_count"] == 1
|
||||
|
||||
|
||||
def test_capture_receipt_does_not_cover_an_unattributed_action_in_the_same_step():
|
||||
value = recorded_sources(
|
||||
[{"step_id": "s", "action_taken": {"action": "tap"}, "generic_tools": [android_receipt()]}]
|
||||
)
|
||||
assert value["sources"][0]["kind"] == "android"
|
||||
assert value["unattributed_step_count"] == 1
|
||||
|
||||
|
||||
def test_flash_capture_receipt_replaces_only_its_matching_action_record():
|
||||
tool = android_receipt()
|
||||
action = {"action": "android_file", "args": {"action": "screenshot", "filename": "home.png"}}
|
||||
value = recorded_sources([{"step_id": "s", "action_taken": action, "generic_tools": [tool]}])
|
||||
assert value["sources"][0]["reference_count"] == 1
|
||||
assert value["unattributed_step_count"] == 0
|
||||
action["args"]["filename"] = "missing.png"
|
||||
value = recorded_sources([{"step_id": "s", "action_taken": action, "generic_tools": [tool]}])
|
||||
assert value["unattributed_step_count"] == 1
|
||||
|
||||
|
||||
def test_note_and_final_report_actions_do_not_claim_device_evidence_or_missing_sources():
|
||||
value = recorded_sources(
|
||||
[
|
||||
{
|
||||
"step_id": "note",
|
||||
"action_taken": {"action": "save_note", "args": {"content": "device claim"}},
|
||||
},
|
||||
{"step_id": "final", "action_taken": {"action": "report_task_status"}},
|
||||
]
|
||||
)
|
||||
assert value["sources"] == []
|
||||
assert value["unattributed_step_count"] == 0
|
||||
@@ -532,7 +532,7 @@ async def test_assert_failure_yields_failed_tests_but_completed_status(tmp_path)
|
||||
|
||||
# SDK wrap-up: trace suffix distinguishes assertion failures from _PASS
|
||||
assert resolve_trace_suffix("completed", outcome) == "_TESTFAIL"
|
||||
assert resolve_trace_suffix("completed", {"tests": {"failed": 0}}) == "_PASS"
|
||||
assert resolve_trace_suffix("completed", {"tests": {"failed": 0}}) == "_UNVERIFIED"
|
||||
assert resolve_trace_suffix("failed", outcome) == "_FAIL"
|
||||
|
||||
# And the return structure surfaces the machine-readable summary
|
||||
|
||||
@@ -22,14 +22,40 @@ sequence must be described too. Nothing is inferred, the description never
|
||||
reaches the wire, and the wire only ever sees coordinates.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.mcp.action_executor import McpActionExecutor, _ArgError
|
||||
from artemis.mcp.action_types import ActionCode, ActionResult, ObserveResult
|
||||
from artemis.utils.action_target import observation_anchor
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("valid", [True, False])
|
||||
async def test_native_action_source_is_stamped_only_after_dispatch(valid):
|
||||
ctx = SimpleNamespace(
|
||||
device=SimpleNamespace(device_id="selected-phone", device_width=1080, device_height=2400),
|
||||
data_engine=None,
|
||||
)
|
||||
executor = McpActionExecutor(ctx, actuator=Mock())
|
||||
session = Mock()
|
||||
session.call = AsyncMock(
|
||||
return_value=ActionResult(ok=True, code=ActionCode.OK, action="click", message="clicked")
|
||||
)
|
||||
executor._session_or_start = AsyncMock(return_value=session)
|
||||
executor._observe_after_action = AsyncMock(return_value=(ObserveResult(ok=False), None))
|
||||
args = {"target": [100, 200], "target_description": "button"} if valid else {}
|
||||
result = await executor._execute_device_action("click", "click", args, "call", _state())
|
||||
if valid:
|
||||
assert result.metadata["source"]["target_id"] == "selected-phone"
|
||||
session.call.assert_awaited_once()
|
||||
else:
|
||||
assert "source" not in result.metadata
|
||||
session.call.assert_not_awaited()
|
||||
|
||||
|
||||
def _make_executor(width=1080, height=2400):
|
||||
ctx = Mock()
|
||||
ctx.device.device_width = width
|
||||
|
||||
@@ -44,6 +44,20 @@ def test_completed_execution_and_failed_test_are_independent():
|
||||
assert resolve_trace_suffix("completed", outcome("failed")) == "_TESTFAIL"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("status", "suffix"),
|
||||
[
|
||||
("passed", "_PASS"),
|
||||
("failed", "_TESTFAIL"),
|
||||
("blocked", "_BLOCKED"),
|
||||
("inconclusive", "_UNVERIFIED"),
|
||||
],
|
||||
)
|
||||
def test_completed_execution_trace_names_use_the_formal_verdict(status, suffix):
|
||||
assert resolve_trace_suffix("completed", outcome(status)) == suffix
|
||||
assert resolve_trace_suffix("completed", None) == "_UNVERIFIED"
|
||||
|
||||
|
||||
def test_skipped_review_metadata_survives_reload_without_changing_model_schema(tmp_path):
|
||||
result = CaseTestResult(
|
||||
status="inconclusive", reason="执行完成 · 未做 AI 判定", verification_skipped=True
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Model HTTP clients consistently opt out of ambient proxies by default."""
|
||||
|
||||
import ssl
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.config.settings import Settings, settings
|
||||
from artemis.llm.router import ModelEndpoint, ModelFactory
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("use_proxy", [False, True])
|
||||
async def test_sync_and_async_clients_follow_env_only_proxy_policy(monkeypatch, use_proxy):
|
||||
monkeypatch.setenv("HTTPS_PROXY", "http://127.0.0.1:9")
|
||||
monkeypatch.setenv("HTTP_PROXY", "http://127.0.0.1:9")
|
||||
monkeypatch.setenv("OPENAI_PROXY", "http://127.0.0.1:9")
|
||||
monkeypatch.delenv("ARTEMIS_FAKE_LLM", raising=False)
|
||||
monkeypatch.setattr(settings, "OPENAI_USE_SYSTEM_PROXY", use_proxy)
|
||||
model = ModelFactory.create_model(
|
||||
ModelEndpoint(api_key="test-only", api_base="https://example.invalid/v1")
|
||||
)
|
||||
try:
|
||||
for client in (model.http_client, model.http_async_client):
|
||||
assert client.trust_env is use_proxy
|
||||
assert bool([mount for mount in client._mounts.values() if mount]) is use_proxy
|
||||
assert client._transport._pool._ssl_context.verify_mode == ssl.CERT_REQUIRED
|
||||
assert model.openai_proxy == ""
|
||||
finally:
|
||||
model.http_client.close()
|
||||
await model.http_async_client.aclose()
|
||||
|
||||
|
||||
def test_proxy_setting_is_env_only_and_part_of_client_cache_identity(monkeypatch):
|
||||
from apps.admin_console.services.model_settings_service import (
|
||||
ModelSettingsUpdate,
|
||||
get_model_settings,
|
||||
)
|
||||
|
||||
monkeypatch.delenv("OPENAI_USE_SYSTEM_PROXY", raising=False)
|
||||
assert Settings(_env_file=None).OPENAI_USE_SYSTEM_PROXY is False
|
||||
monkeypatch.setenv("OPENAI_USE_SYSTEM_PROXY", "true")
|
||||
assert Settings(_env_file=None).OPENAI_USE_SYSTEM_PROXY is True
|
||||
endpoint = ModelEndpoint(api_key="test-only")
|
||||
monkeypatch.setattr(settings, "OPENAI_USE_SYSTEM_PROXY", False)
|
||||
direct = endpoint.cache_key()
|
||||
monkeypatch.setattr(settings, "OPENAI_USE_SYSTEM_PROXY", True)
|
||||
assert endpoint.cache_key() != direct
|
||||
assert not any("proxy" in key.lower() for key in get_model_settings())
|
||||
assert not any("proxy" in key.lower() for key in ModelSettingsUpdate.model_fields)
|
||||
@@ -892,6 +892,16 @@ async def test_workspace_computer_discovery_never_queries_adb(monkeypatch):
|
||||
assert set(routes) == {"/api/status", "/api/plugins"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_computer_task_uses_the_actual_host_platform():
|
||||
from artemis.platform import platform
|
||||
from artemis.sdk.agent import Agent
|
||||
|
||||
agent = Agent()
|
||||
await agent.init(execution_target="computer")
|
||||
assert agent._device_context.host_platform == platform.os_type.name
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sdk_setup_failure_releases_plugin_version_pin(store, monkeypatch):
|
||||
from artemis.sdk.agent import Agent
|
||||
|
||||
@@ -126,6 +126,14 @@ def test_record_llm_usage_accumulates_across_calls_same_session():
|
||||
assert meter.last_prompt_tokens == 250
|
||||
|
||||
|
||||
def test_record_llm_usage_preserves_the_call_model():
|
||||
engine = Mock()
|
||||
engine.current_session_id = uuid4()
|
||||
engine.current_step_id = None
|
||||
payload = record_llm_usage(engine, _msg_with_usage(), model="gpt-6.1-terra")
|
||||
assert payload["model"] == "gpt-6.1-terra"
|
||||
|
||||
|
||||
def test_record_llm_usage_never_raises_and_skips_gracefully():
|
||||
# No engine
|
||||
assert record_llm_usage(None, _msg_with_usage()) is None
|
||||
|
||||
Reference in new issue
Block a user