feat: release 2.4.0 with original cases and automatic human review

This commit is contained in:
chenmingxuan committed 2026-10-28 17:25:08 +08:00
1 parent baf069854b
commit 4dc2057d43
66 files changed
+1390 -206

No files matched your search

@@ -908,6 +908,85 @@ def test_reports_do_not_execute_formulas_or_invent_pass(isolated):
assert "未验证".encode() in doc.read("word/document.xml")
@pytest.mark.parametrize("result_status", ["passed", "failed", "blocked", "inconclusive"])
def test_report_without_output_body_uses_effective_verdict_and_evidence(isolated, result_status):
job = isolated.jobs(service.create_batch(payload())["id"])[0]
isolated.save_job({**job, "status": "completed"})
folder = reports.TRACES_PATH / job["session_id"]
folder.mkdir(parents=True)
(folder / "run_outcome.json").write_text(
json.dumps(
{
"task_status": "completed",
"test_result": {
"status": result_status,
"reason": "实际验收依据",
"evidence": ["记录中的证据"],
},
}
),
encoding="utf-8",
)
report = reports.session_report(job["session_id"])
assert report["verdict"] in report["content"]
assert "实际验收依据" in report["content"]
assert "记录中的证据" in report["content"]
assert "未保存独立输出正文" in report["content"]
assert "没有保存最终结论" not in report["content"]
assert report["test_result"]["status"] == result_status
def test_report_without_output_body_keeps_human_review_authoritative(isolated):
job = isolated.jobs(service.create_batch(payload())["id"])[0]
isolated.save_job({**job, "status": "completed"})
folder = reports.TRACES_PATH / job["session_id"]
folder.mkdir(parents=True)
(folder / "run_outcome.json").write_text(
json.dumps(
{
"test_result": {"status": "passed", "reason": "AI 原判通过"},
}
),
encoding="utf-8",
)
isolated.save_case_review(job["session_id"], "failed", "用户确认实际结果不符合预期")
report = reports.session_report(job["session_id"])
assert "人工确认" in report["content"]
assert "用户确认实际结果不符合预期" in report["content"]
assert "AI 原判通过" not in report["content"]
assert report["test_result"]["status"] == "failed"
def test_report_projects_stored_android_receipts_without_rewriting_history(isolated):
from tests.unit.data_engine.test_evidence_sources import android_receipt
sid = isolated.jobs(service.create_batch(payload())["id"])[0]["session_id"]
receipt = android_receipt()["payload"]["result"]
raw_payload = json.dumps({"args": {"action": "screenshot"}, "result": json.dumps(receipt)})
with sqlite3.connect(isolated.path) as conn:
conn.execute(
"INSERT INTO sessions(session_id,initial_goal,start_time,status) VALUES (?,?,?,?)",
(sid, "跨端截图", 1, "completed"),
)
conn.execute(
"INSERT INTO steps(step_id,session_id,step_number) VALUES (?,?,?)", ("s", sid, 1)
)
conn.execute(
"INSERT INTO traces(trace_id,session_id,step_id,type,name,timestamp,status,payload) VALUES (?,?,?,?,?,?,?,?)",
("capture", sid, "s", "tool", "android_file", 2, "success", raw_payload),
)
report = reports.session_report(sid)
source = report["evidence_sources"]["sources"][0]
assert source["kind"] == "android"
assert source["references"][0]["trace_id"] == "capture"
assert report["evidence_sources"]["unattributed_step_count"] == 0
with sqlite3.connect(isolated.path) as conn:
assert (
conn.execute("SELECT payload FROM traces WHERE trace_id='capture'").fetchone()[0]
== raw_payload
)
def test_failed_report_exposes_logged_reason_and_formats_chinese_markdown(isolated):
job = isolated.jobs(service.create_batch(payload())["id"])[0]
isolated.save_job({**job, "status": "failed"})
@@ -110,6 +110,8 @@ def test_strip_trace_status_suffix_handles_every_terminal_marker():
assert MediaService.strip_trace_status_suffix("web_1_PASS_2026") == "web_1"
assert MediaService.strip_trace_status_suffix("web_1_FAIL_2026") == "web_1"
assert MediaService.strip_trace_status_suffix("web_1_TESTFAIL_2026") == "web_1"
assert MediaService.strip_trace_status_suffix("web_1_BLOCKED_2026") == "web_1"
assert MediaService.strip_trace_status_suffix("web_1_UNVERIFIED_2026") == "web_1"
assert MediaService.strip_trace_status_suffix("web_1") == "web_1"
@@ -44,6 +44,10 @@ async def test_list_sessions_uses_batched_history_metadata(monkeypatch):
repo.get_llm_traces_for_profiles_map.return_value = {
"session-3": ['{"model": "gemini-3.7-pro"}']
}
repo.get_recorded_model_info_map.return_value = {
"session-1": {"id": "gpt-6.1-terra", "provider": "openai"},
"session-2": {"id": "gpt-6-luna", "provider": "openai"},
}
repo.get_agent_trace_names_map.return_value = {}
monkeypatch.setattr(sessions_router, "session_repo", repo, raising=False)
@@ -57,6 +61,10 @@ async def test_list_sessions_uses_batched_history_metadata(monkeypatch):
result = await sessions_router.list_sessions()
assert len(result) == 3
assert [row["model_info"]["id"] for row in result] == ["gpt-6.1-terra", "gpt-6-luna", ""]
repo.get_recorded_model_info_map.assert_called_once_with(
["session-1", "session-2", "session-3"]
)
repo.get_latest_video_recordings_map.assert_called_once_with()
repo.get_llm_traces_for_profiles_map.assert_called_once_with(["session-3"])
repo.get_agent_trace_names_map.assert_called_once_with()
@@ -1,3 +1,4 @@
import json
from unittest.mock import patch
from apps.admin_console.database.connection import db_session
@@ -37,3 +38,31 @@ def test_session_list_harvest_rechecks_worker_liveness(tmp_path):
assert repository.harvest_orphaned_sessions(["external-live"]) == 0
assert repository.get_session_status("external-live") == "running"
def test_recorded_model_metadata_is_per_session_and_prefers_operator(tmp_path):
db_path = tmp_path / "sessions.db"
repository = SessionRepository(db_path)
for session_id in ("one", "two", "no-record"):
_insert_running_session(db_path, session_id, 1)
with db_session(db_path) as conn:
for trace_id, session_id, timestamp, payload in (
("invalid", "one", 0, "{"),
("planner", "one", 1, json.dumps({"role": "planner", "model": "other-model"})),
(
"operator",
"one",
2,
json.dumps({"role": "operator", "model": "gpt-6.1-terra", "provider": "openai"}),
),
("flash", "two", 1, json.dumps({"role": "FlashRunner", "model": "gpt-6-luna"})),
):
conn.execute(
"INSERT INTO traces(trace_id,session_id,type,timestamp,payload) VALUES (?,?,?,?,?)",
(trace_id, session_id, "llm_diagnostic", timestamp, payload),
)
conn.commit()
assert repository.get_recorded_model_info_map(["one", "two", "no-record"]) == {
"one": {"id": "gpt-6.1-terra", "provider": "openai"},
"two": {"id": "gpt-6-luna", "provider": ""},
}
@@ -99,6 +99,34 @@ def test_usage_flash_runner_counts_as_operator_and_has_no_tuning(tmp_path):
assert usage["run_tuning"] is None
def test_usage_uses_its_recorded_model_and_supports_legacy_endpoint_keys(tmp_path):
db_path = tmp_path / "sessions.db"
repo = SessionRepository(db_path)
_insert_session(db_path, "models", {"profile": "flash"})
_insert_usage(db_path, "models", "new", 1.0, {**_usage(100, 10), "model": "terra"})
_insert_usage(
db_path,
"models",
"legacy",
2.0,
{**_usage(200, 20), "source": "openai:terra:https://example.test:high"},
)
_insert_usage(
db_path, "models", "lens", 3.0, {**_usage(300, 30), "source": "lens:step_capsule:terra"}
)
_insert_usage(db_path, "models", "unknown", 4.0, {**_usage(300, 30), "source": "other"})
_insert_usage(
db_path,
"models",
"fallback",
5.0,
{**_usage(300, 30), "source": "lens:step_capsule:fallback"},
)
models = {item["model"]: item for item in repo.get_session_usage("models")["by_node_model"]}
assert models["terra"]["llm_calls"] == 3
assert models["unknown"]["llm_calls"] == 2
def test_usage_for_unknown_or_empty_session_is_zeroed(tmp_path):
db_path = tmp_path / "sessions.db"
repo = SessionRepository(db_path)
@@ -112,6 +140,26 @@ def test_usage_for_unknown_or_empty_session_is_zeroed(tmp_path):
assert usage["run_tuning"] is None
def test_legacy_usage_preserves_model_colons_and_keeps_models_separate(tmp_path):
db_path = tmp_path / "sessions.db"
repo = SessionRepository(db_path)
_insert_session(db_path, "models", {"profile": "flash"})
sources = {
"team/model:stable": "openai:team/model:stable:https://example.test:8443/v1:high",
"team/model:preview": "openai:team/model:preview:http://[::1]:8080/v1:high",
"unknown": "openai:team/model:preview:unparseable:high",
}
for index, source in enumerate(sources.values()):
_insert_usage(
db_path, "models", str(index), float(index), {**_usage(100, 10), "source": source}
)
groups = {item["model"]: item for item in repo.get_session_usage("models")["by_node_model"]}
assert groups.keys() == sources.keys()
assert all(item["llm_calls"] == 1 for item in groups.values())
assert all(item["prompt_tokens"] == 100 for item in groups.values())
def test_usage_route_is_registered():
from apps.admin_console.routers import sessions
@@ -3,6 +3,30 @@ import json
from apps.admin_console.database.repositories.step_repository import StepRepository
def test_android_capture_receipt_survives_cleaning_without_becoming_image_store_id():
receipt = {
"ok": True,
"action": "screenshot",
"verified": True,
"device_serial": "device-under-test",
"trace_path": "artifacts/android/screen.png",
"sha256": "a" * 64,
"captured_at": "2026-10-28T00:00:00Z",
"content": "large file content must not be copied into the task stream",
}
cleaned = StepRepository()._clean_tool_payload(
{"args": {"action": "screenshot"}, "result": json.dumps(receipt)},
"tool",
"android_file",
)
assert cleaned["result"] == {key: value for key, value in receipt.items() if key != "content"}
assert "post_image_name" not in cleaned
wrapped = StepRepository()._clean_tool_payload(
{"result": json.dumps({"outcome": json.dumps(receipt)})}, "tool", "android_file"
)
assert wrapped["result"] == cleaned["result"]
def test_legacy_pause_log_is_normalized_to_failed_llm_trace():
repository = StepRepository()
trace = {
@@ -57,10 +57,13 @@ def manifest_payload():
}
def install(plugins):
def install(plugins, original_case=False):
package = BytesIO()
manifest = manifest_payload()
if original_case:
manifest["workspace"]["panels"][0]["kind"] = "original_case"
with zipfile.ZipFile(package, "w") as archive:
archive.writestr("plugin.json", json.dumps(manifest_payload()))
archive.writestr("plugin.json", json.dumps(manifest))
install_archive(plugins, package.getvalue())
plugins.set_enabled("review", True)
@@ -105,6 +108,86 @@ def save(client, values, revision=0, request_id=None):
)
def test_original_case_import_requires_enabled_plugin_and_keeps_execution_prompt(context):
from apps.admin_console.services import task_import
store, plugins, client = context
legacy = load_workbook(BytesIO(task_import.template()))
assert legacy["任务"].max_column == 7
legacy["任务"].append(["OLD", "", "", "旧用例", "", "原样执行", ""])
legacy_bytes = BytesIO()
legacy.save(legacy_bytes)
assert "original_case" not in task_import.parse_workbook(legacy_bytes.getvalue())["tasks"][0]
install(plugins, original_case=True)
workbook = load_workbook(BytesIO(task_import.template()))
assert [cell.value for cell in workbook["任务"][1]] == [
*task_import.HEADERS,
*task_import.CASE_HEADERS,
]
workbook["任务"].append(
[
"CASE-001",
"检测项目",
"测试子项",
"测试标题",
"提取的约束",
"只执行此提示词",
"{}",
"原始预置条件",
"1. 第一步\n2. 第二步",
"原始预期结果",
]
)
stream = BytesIO()
workbook.save(stream)
case = task_import.parse_workbook(stream.getvalue())["tasks"][0]
assert task_import.case_goal(case) == "只执行此提示词"
assert case["preconditions"] == "提取的约束"
assert case["original_case"] == {
"preconditions": "原始预置条件",
"steps": "1. 第一步\n2. 第二步",
"expected": "原始预期结果",
}
assert task_import.parse_workbook(legacy_bytes.getvalue())["valid"]
store.save_job({"session_id": "task-one", "status": "completed", "case": case})
result = client.get("/api/task-extensions/task-one").json()
assert "CASE-001" in result["case_title"] and "测试标题" in result["case_title"]
assert result["original_case"] == case["original_case"]
assert result["initial_prompt"] == "只执行此提示词"
assert report_export.session_report("task-one")["original_case"] == case["original_case"]
plugins.set_enabled("review", False)
assert load_workbook(BytesIO(task_import.template()))["任务"].max_column == 7
with pytest.raises(ValueError, match="列名或顺序"):
task_import.parse_workbook(stream.getvalue())
def test_review_navigation_uses_batch_order_and_excludes_unfinished_jobs(context):
store, _, client = context
store.create_batch({"id": "review-batch", "status": "completed"}, [], {})
for position, status in [(3, "completed"), (1, "failed"), (2, "pending")]:
store.save_job(
{
"session_id": f"batch-task-{position}",
"batch_id": "review-batch",
"position": position,
"status": status,
"case": {"name": f"用例{position}", "goal": "测试"},
}
)
first = client.get("/api/task-extensions/batch-task-1").json()["navigation"]
assert first == {
"scope": "batch",
"position": 1,
"total": 2,
"previous": None,
"next": "batch-task-3",
}
last = client.get("/api/task-extensions/batch-task-3").json()["navigation"]
assert last["previous"] == "batch-task-1" and last["next"] is None
def test_workspace_requires_v3_and_valid_bounded_declarations():
for change in (
{"schema_version": 2},
@@ -601,6 +601,45 @@ async def test_status_reports_external_global_owner_without_ipc_connection():
assert result["goal"] == "CLI task: inspect settings"
@pytest.mark.asyncio
@pytest.mark.parametrize("recorded", [True, False])
async def test_status_uses_the_active_sessions_model(recorded):
owner = DeviceLockOwner(
pid=24680,
process_created_at=1234.5,
token="active-owner-token",
device_id="emulator-5554",
description="active task",
acquired_at="2026-08-24T00:00:00+00:00",
session_id="active",
ingress="web",
)
state.queue_items.append(
{"session_id": "active", "status": "running", "runtime_overrides": {"model": "terra"}}
)
with (
patch.object(TaskQueueService, "ensure_worker_running"),
patch(
"apps.admin_console.routers.tasks.DeviceExecutionLock.get_active_owner",
return_value=owner,
),
patch(
"apps.admin_console.routers.tasks.DeviceExecutionLock.get_active_owners",
return_value={},
),
patch("apps.admin_console.routers.tasks.session_repo") as repo,
patch("apps.admin_console.routers.tasks.model_service") as models,
):
repo.get_latest_session.return_value = None
repo.get_session_by_id.return_value = None
repo.get_recorded_model_info_map.return_value = (
{"active": {"id": "terra-recorded", "provider": "openai"}} if recorded else {}
)
models.get_active_model_info.return_value = {"id": "global", "provider": "openai"}
result = await get_status()
assert result["model_info"]["id"] == ("terra-recorded" if recorded else "terra")
@pytest.mark.asyncio
async def test_cancel_task_triggers_next_pending_task():
executed_goals = []
+5 -1
View File
@@ -379,9 +379,13 @@ def test_flash_records_the_executors_target_semantics(mock_context):
runner,
"click",
click_args,
metadata={"target_semantics": {"target_description": "play button"}},
metadata={
"target_semantics": {"target_description": "play button"},
"source": {"kind": "android", "target_id": "selected-phone", "label": "安卓屏幕"},
},
)
assert click["target_description"] == "play button"
assert click["source"]["target_id"] == "selected-phone"
assert "target_text" not in click
assert format_actions_clean(click) == "Tapped 'play button' (self-described) at [320, 399]"
+5
View File
@@ -124,6 +124,11 @@ async def test_planner_knows_computer_plugins_with_mobile_primary_screen(mock_co
content = llm.astream.call_args.args[0][1].content
text = "\n".join(part["text"] for part in content if part["type"] == "text")
assert text.count("电脑插件:cua-driver") == 1
assert "规划器只使用本阶段绑定的笔记工具" in text
assert "不表示执行器缺少这些能力" in text
assert "实际加载与操作回执" in text
bound_names = {tool.name for tool in llm.bind_tools.call_args.args[0]}
assert "plugin_load" not in bound_names
assert f"当前主设备截图({'电脑' if target == 'computer' else '安卓'})" in text
assert "不决定其他步骤的执行平台" in text
+6
View File
@@ -122,6 +122,7 @@ def mock_mcp():
async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
"""Test that ValidatorNode executes actions successfully."""
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
mock_context.device.device_id = "selected-phone"
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
state.initial_goal = "在真实审批表单上 Approve once,不要普通聊天代答或重复提交。"
@@ -142,6 +143,11 @@ async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "tap"
assert "attempts" not in report["execution"][0]
assert report["execution"][0]["source"] == {
"kind": "android",
"target_id": "selected-phone",
"label": "安卓屏幕",
}
# The Operator's tap verb was translated to the canonical click tool.
assert len(mock_mcp.calls_for("click")) == 1
@@ -1,5 +1,7 @@
from copy import deepcopy
import pytest
from artemis.data_engine.evidence_sources import recorded_sources
@@ -59,6 +61,21 @@ def test_sources_remain_separate_and_references_use_host_ids():
assert value["sources"][1]["category"] == "interface"
def test_window_label_change_does_not_attribute_all_steps_to_the_first_title():
first = record("computer", "window", call_id="c1")
second = record("computer", "window", call_id="c2")
first["source"]["label"] = "first page"
second["source"]["label"] = "second page"
result = recorded_sources(
[{"step_id": "s1", "action_taken": first}, {"step_id": "s2", "action_taken": second}]
)
source = result["sources"][0]
assert result["total_sources"] == 1
assert source["reference_count"] == 2
assert source["label"] == ""
assert source["label_varies"] is True
def test_model_arguments_and_unsourced_images_do_not_fabricate_source_metadata():
value = recorded_sources(
[
@@ -103,3 +120,98 @@ def test_partial_source_metadata_does_not_hide_unknown_actions_in_the_same_step(
)
assert value["total_sources"] == 1
assert value["unattributed_step_count"] == 1
def android_receipt():
return {
"trace_id": "capture-trace",
"type": "tool",
"name": "android_file",
"status": "success",
"payload": {
"result": {
"ok": True,
"action": "screenshot",
"verified": True,
"device_serial": "test-phone",
"trace_path": "artifacts/android/home.png",
"sha256": "a" * 64,
"captured_at": "2026-10-28T00:00:00Z",
}
},
}
def test_android_receipts_join_computer_sources_and_deduplicate_nested_tool_traces():
tool = android_receipt()
duplicate = {**deepcopy(tool), "trace_id": "outer-wrapper"}
steps = [
{
"step_id": "mixed",
"action_taken": record("computer", "window"),
"generic_tools": [tool, duplicate],
},
{"step_id": "final", "generic_tools": [tool]},
{"step_id": "empty", "action_taken": None, "generic_tools": []},
]
before = deepcopy(steps)
result = recorded_sources(steps)
sources = {source["kind"]: source for source in result["sources"]}
assert set(sources) == {"android", "computer"}
assert sources["android"]["target_id"] == "test-phone"
assert sources["android"]["reference_count"] == 2
assert sources["android"]["references"][0]["trace_id"] == "capture-trace"
assert sources["android"]["references"][0]["artifact_path"] == "artifacts/android/home.png"
assert result["unattributed_step_count"] == 0
assert steps == before
@pytest.mark.parametrize("invalid", ["failed", "unverified", "different_tool", "arguments_only"])
def test_only_successful_android_capture_receipts_supply_provenance(invalid):
tool = android_receipt()
if invalid == "failed":
tool["status"] = "failed"
elif invalid == "unverified":
tool["payload"]["result"]["verified"] = False
elif invalid == "different_tool":
tool["name"] = "read_note"
else:
tool["payload"] = {"args": tool["payload"]["result"]}
value = recorded_sources(
[{"step_id": "s", "action_taken": {"action": "tap"}, "generic_tools": [tool]}]
)
assert value["sources"] == []
assert value["unattributed_step_count"] == 1
def test_capture_receipt_does_not_cover_an_unattributed_action_in_the_same_step():
value = recorded_sources(
[{"step_id": "s", "action_taken": {"action": "tap"}, "generic_tools": [android_receipt()]}]
)
assert value["sources"][0]["kind"] == "android"
assert value["unattributed_step_count"] == 1
def test_flash_capture_receipt_replaces_only_its_matching_action_record():
tool = android_receipt()
action = {"action": "android_file", "args": {"action": "screenshot", "filename": "home.png"}}
value = recorded_sources([{"step_id": "s", "action_taken": action, "generic_tools": [tool]}])
assert value["sources"][0]["reference_count"] == 1
assert value["unattributed_step_count"] == 0
action["args"]["filename"] = "missing.png"
value = recorded_sources([{"step_id": "s", "action_taken": action, "generic_tools": [tool]}])
assert value["unattributed_step_count"] == 1
def test_note_and_final_report_actions_do_not_claim_device_evidence_or_missing_sources():
value = recorded_sources(
[
{
"step_id": "note",
"action_taken": {"action": "save_note", "args": {"content": "device claim"}},
},
{"step_id": "final", "action_taken": {"action": "report_task_status"}},
]
)
assert value["sources"] == []
assert value["unattributed_step_count"] == 0
+1 -1
View File
@@ -532,7 +532,7 @@ async def test_assert_failure_yields_failed_tests_but_completed_status(tmp_path)
# SDK wrap-up: trace suffix distinguishes assertion failures from _PASS
assert resolve_trace_suffix("completed", outcome) == "_TESTFAIL"
assert resolve_trace_suffix("completed", {"tests": {"failed": 0}}) == "_PASS"
assert resolve_trace_suffix("completed", {"tests": {"failed": 0}}) == "_UNVERIFIED"
assert resolve_trace_suffix("failed", outcome) == "_FAIL"
# And the return structure surfaces the machine-readable summary
@@ -22,14 +22,40 @@ sequence must be described too. Nothing is inferred, the description never
reaches the wire, and the wire only ever sees coordinates.
"""
from unittest.mock import Mock
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
import pytest
from artemis.mcp.action_executor import McpActionExecutor, _ArgError
from artemis.mcp.action_types import ActionCode, ActionResult, ObserveResult
from artemis.utils.action_target import observation_anchor
@pytest.mark.asyncio
@pytest.mark.parametrize("valid", [True, False])
async def test_native_action_source_is_stamped_only_after_dispatch(valid):
ctx = SimpleNamespace(
device=SimpleNamespace(device_id="selected-phone", device_width=1080, device_height=2400),
data_engine=None,
)
executor = McpActionExecutor(ctx, actuator=Mock())
session = Mock()
session.call = AsyncMock(
return_value=ActionResult(ok=True, code=ActionCode.OK, action="click", message="clicked")
)
executor._session_or_start = AsyncMock(return_value=session)
executor._observe_after_action = AsyncMock(return_value=(ObserveResult(ok=False), None))
args = {"target": [100, 200], "target_description": "button"} if valid else {}
result = await executor._execute_device_action("click", "click", args, "call", _state())
if valid:
assert result.metadata["source"]["target_id"] == "selected-phone"
session.call.assert_awaited_once()
else:
assert "source" not in result.metadata
session.call.assert_not_awaited()
def _make_executor(width=1080, height=2400):
ctx = Mock()
ctx.device.device_width = width
+14
View File
@@ -44,6 +44,20 @@ def test_completed_execution_and_failed_test_are_independent():
assert resolve_trace_suffix("completed", outcome("failed")) == "_TESTFAIL"
@pytest.mark.parametrize(
("status", "suffix"),
[
("passed", "_PASS"),
("failed", "_TESTFAIL"),
("blocked", "_BLOCKED"),
("inconclusive", "_UNVERIFIED"),
],
)
def test_completed_execution_trace_names_use_the_formal_verdict(status, suffix):
assert resolve_trace_suffix("completed", outcome(status)) == suffix
assert resolve_trace_suffix("completed", None) == "_UNVERIFIED"
def test_skipped_review_metadata_survives_reload_without_changing_model_schema(tmp_path):
result = CaseTestResult(
status="inconclusive", reason="执行完成 · 未做 AI 判定", verification_skipped=True
+49
View File
@@ -0,0 +1,49 @@
"""Model HTTP clients consistently opt out of ambient proxies by default."""
import ssl
import pytest
from artemis.config.settings import Settings, settings
from artemis.llm.router import ModelEndpoint, ModelFactory
@pytest.mark.asyncio
@pytest.mark.parametrize("use_proxy", [False, True])
async def test_sync_and_async_clients_follow_env_only_proxy_policy(monkeypatch, use_proxy):
monkeypatch.setenv("HTTPS_PROXY", "http://127.0.0.1:9")
monkeypatch.setenv("HTTP_PROXY", "http://127.0.0.1:9")
monkeypatch.setenv("OPENAI_PROXY", "http://127.0.0.1:9")
monkeypatch.delenv("ARTEMIS_FAKE_LLM", raising=False)
monkeypatch.setattr(settings, "OPENAI_USE_SYSTEM_PROXY", use_proxy)
model = ModelFactory.create_model(
ModelEndpoint(api_key="test-only", api_base="https://example.invalid/v1")
)
try:
for client in (model.http_client, model.http_async_client):
assert client.trust_env is use_proxy
assert bool([mount for mount in client._mounts.values() if mount]) is use_proxy
assert client._transport._pool._ssl_context.verify_mode == ssl.CERT_REQUIRED
assert model.openai_proxy == ""
finally:
model.http_client.close()
await model.http_async_client.aclose()
def test_proxy_setting_is_env_only_and_part_of_client_cache_identity(monkeypatch):
from apps.admin_console.services.model_settings_service import (
ModelSettingsUpdate,
get_model_settings,
)
monkeypatch.delenv("OPENAI_USE_SYSTEM_PROXY", raising=False)
assert Settings(_env_file=None).OPENAI_USE_SYSTEM_PROXY is False
monkeypatch.setenv("OPENAI_USE_SYSTEM_PROXY", "true")
assert Settings(_env_file=None).OPENAI_USE_SYSTEM_PROXY is True
endpoint = ModelEndpoint(api_key="test-only")
monkeypatch.setattr(settings, "OPENAI_USE_SYSTEM_PROXY", False)
direct = endpoint.cache_key()
monkeypatch.setattr(settings, "OPENAI_USE_SYSTEM_PROXY", True)
assert endpoint.cache_key() != direct
assert not any("proxy" in key.lower() for key in get_model_settings())
assert not any("proxy" in key.lower() for key in ModelSettingsUpdate.model_fields)
+10
View File
@@ -892,6 +892,16 @@ async def test_workspace_computer_discovery_never_queries_adb(monkeypatch):
assert set(routes) == {"/api/status", "/api/plugins"}
@pytest.mark.asyncio
async def test_computer_task_uses_the_actual_host_platform():
from artemis.platform import platform
from artemis.sdk.agent import Agent
agent = Agent()
await agent.init(execution_target="computer")
assert agent._device_context.host_platform == platform.os_type.name
@pytest.mark.asyncio
async def test_sdk_setup_failure_releases_plugin_version_pin(store, monkeypatch):
from artemis.sdk.agent import Agent
+8
View File
@@ -126,6 +126,14 @@ def test_record_llm_usage_accumulates_across_calls_same_session():
assert meter.last_prompt_tokens == 250
def test_record_llm_usage_preserves_the_call_model():
engine = Mock()
engine.current_session_id = uuid4()
engine.current_step_id = None
payload = record_llm_usage(engine, _msg_with_usage(), model="gpt-6.1-terra")
assert payload["model"] == "gpt-6.1-terra"
def test_record_llm_usage_never_raises_and_skips_gracefully():
# No engine
assert record_llm_usage(None, _msg_with_usage()) is None