chenmingxuan
e1ea099592
Install and update artemis-use with task diagnostics and plugin guidance. Preserve cross-device file receipts, review canonical reports, guard stage transitions and reduce small history summaries. Improve exploration lifecycle, request recovery and browser live connections.
258 lines
10 KiB
Python
258 lines
10 KiB
Python
"""Read-only diagnostics and destructive-action gates against disposable task data."""
|
|
|
|
import json
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, Mock
|
|
from uuid import uuid4
|
|
|
|
from fastapi import FastAPI
|
|
from fastapi.testclient import TestClient
|
|
import pytest
|
|
|
|
from apps.admin_console.database.repositories.session_repository import SessionRepository
|
|
from apps.admin_console.routers import sessions
|
|
from apps.admin_console.services import report_export, task_deletion, task_inspection
|
|
from apps.admin_console.services.automation_store import AutomationStore
|
|
from apps.admin_console.services.request_diagnostics import request_diagnostics
|
|
from artemis.data_engine.storage import StorageManager
|
|
from artemis.runtime import trace_store
|
|
from mcp_server.tools import automation, exploration, inspect_trace, task_manager
|
|
|
|
|
|
@pytest.fixture
|
|
def task(tmp_path, monkeypatch):
|
|
sid = str(uuid4())
|
|
db = tmp_path / "data_engine.db"
|
|
StorageManager(db, tmp_path)
|
|
store = AutomationStore(db)
|
|
repo = SessionRepository(db)
|
|
with store.connection() as connection:
|
|
connection.execute(
|
|
"INSERT INTO sessions (session_id, initial_goal, start_time, status, device_info) "
|
|
"VALUES (?, 'observe', 1, 'completed', '{}')",
|
|
(sid,),
|
|
)
|
|
connection.commit()
|
|
monkeypatch.setattr(task_inspection, "session_repo", repo)
|
|
monkeypatch.setattr(task_inspection, "store", store)
|
|
monkeypatch.setattr(task_inspection.evidence_service, "task_root", lambda _: tmp_path / sid)
|
|
monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path))
|
|
monkeypatch.setattr(report_export, "store", store)
|
|
monkeypatch.setattr(report_export, "TRACES_PATH", tmp_path)
|
|
(tmp_path / sid).mkdir()
|
|
return sid, db, tmp_path, store
|
|
|
|
|
|
def add_trace(store, sid, kind, name, payload, timestamp=1):
|
|
with store.connection() as connection:
|
|
connection.execute(
|
|
"INSERT INTO traces (trace_id,session_id,type,name,payload,timestamp) VALUES (?,?,?,?,?,?)",
|
|
(str(uuid4()), sid, kind, name, json.dumps(payload), timestamp),
|
|
)
|
|
connection.commit()
|
|
|
|
|
|
def test_usage_uses_input_denominator_and_does_not_invent_missing_model(task):
|
|
sid, _, _, store = task
|
|
assert task_inspection.inspect_task(sid, "usage")["cache_hit_rate"] is None
|
|
add_trace(
|
|
store,
|
|
sid,
|
|
"llm_call",
|
|
"llm_usage",
|
|
{
|
|
"node": "operator",
|
|
"model": "model-a",
|
|
"prompt_tokens": 1000,
|
|
"cached_tokens": 900,
|
|
"completion_tokens": 500,
|
|
},
|
|
)
|
|
add_trace(
|
|
store,
|
|
sid,
|
|
"llm_call",
|
|
"llm_usage",
|
|
{
|
|
"node": "checker",
|
|
"prompt_tokens": 1000,
|
|
"cached_tokens": 0,
|
|
"completion_tokens": 500,
|
|
},
|
|
)
|
|
result = task_inspection.inspect_task(sid, "usage")
|
|
assert result["cache_hit_rate"] == 0.45
|
|
assert result["total_tokens"] == 3000
|
|
assert result["by_node_model"][1]["model"] == "unknown"
|
|
|
|
|
|
def test_log_tail_is_bounded_paginated_and_redacted(task, monkeypatch):
|
|
sid, _, root, _ = task
|
|
monkeypatch.setenv("TEST_API_KEY", "test-only-sensitive-value")
|
|
lines = ["x" * 200 for _ in range(1000)] + [
|
|
"old",
|
|
"test-only-sensitive-value",
|
|
"Bearer abcdef",
|
|
"password=hush",
|
|
]
|
|
(root / sid / "stderr.log").write_text("\n".join(lines), "utf-8")
|
|
result = task_inspection.inspect_task(sid, "logs", limit=2)
|
|
error = result["logs"][1]
|
|
assert error["older_bytes_omitted"] > 0
|
|
assert error["next_offset"] == 2
|
|
assert error["lines"] == ["Bearer [REDACTED]", "password=[REDACTED]"]
|
|
older = task_inspection.inspect_task(sid, "logs", offset=2, limit=2)
|
|
assert older["logs"][1]["lines"] == ["old", "[REDACTED]"]
|
|
assert result["logs"][0]["available"] is False
|
|
|
|
|
|
def test_report_pagination_preserves_total_and_caps_nested_content(task, monkeypatch):
|
|
sid, _, _, _ = task
|
|
monkeypatch.setattr(
|
|
report_export,
|
|
"session_report",
|
|
lambda _: {
|
|
"steps": [{"number": i} for i in range(35)],
|
|
"content": "x" * 100000,
|
|
"metadata": {"api_key": "hidden"},
|
|
"nested": [[{"value": "x" * 100} for _ in range(1000)] for _ in range(1000)],
|
|
},
|
|
)
|
|
result = task_inspection.inspect_task(sid, "report", offset=20, limit=10)
|
|
assert result["steps"]["items"] == [{"number": i} for i in range(20, 30)]
|
|
assert result["steps"]["total"] == 35
|
|
assert result["steps"]["next_offset"] == 30
|
|
assert result["metadata"]["api_key"] == "[REDACTED]"
|
|
assert result["content_truncated"] is True
|
|
assert len(json.dumps(result, ensure_ascii=False)) < 150000
|
|
|
|
|
|
def test_request_pagination_reports_latest_phase_only(task):
|
|
sid, db, _, store = task
|
|
with store.connection() as connection:
|
|
connection.execute("UPDATE sessions SET status='running' WHERE session_id=?", (sid,))
|
|
connection.commit()
|
|
add_trace(
|
|
store, sid, "llm_diagnostic", "request", {"request_id": "r", "phase": "waiting_response"}, 1
|
|
)
|
|
add_trace(store, sid, "llm_diagnostic", "request", {"request_id": "r", "phase": "completed"}, 2)
|
|
assert request_diagnostics(sid, db_path=db, limit=1)["next_offset"] == 1
|
|
older = request_diagnostics(sid, db_path=db, limit=1, offset=1)
|
|
assert older["requests"][0]["active"] is False
|
|
assert older["next_offset"] is None
|
|
|
|
|
|
def test_status_and_report_use_same_human_review_without_losing_ai_result(task):
|
|
sid, db, root, store = task
|
|
(root / sid / "run_outcome.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"test_result": {
|
|
"status": "failed",
|
|
"reason": "AI found mismatch",
|
|
"evidence": ["step 1: actual differs from expected"],
|
|
},
|
|
}
|
|
),
|
|
"utf-8",
|
|
)
|
|
store.put(
|
|
"case_review",
|
|
sid,
|
|
{"reviews": [{"status": "passed", "reason": "用户确认", "reviewed_at": 10}]},
|
|
)
|
|
result = task_manager.mobile_manage_task("status", sid)["test_result"]
|
|
assert result == report_export.session_report(sid, db_path=db)["test_result"]
|
|
assert result["source"] == "human"
|
|
assert result["ai_result"]["status"] == "failed"
|
|
|
|
|
|
def test_inspection_not_found_and_pagination_validation(task):
|
|
app = FastAPI()
|
|
app.include_router(sessions.router)
|
|
client = TestClient(app)
|
|
assert client.get(f"/api/sessions/{uuid4()}/inspection").status_code == 404
|
|
assert client.get(f"/api/sessions/{task[0]}/inspection?limit=101").status_code == 422
|
|
|
|
|
|
def test_active_normal_task_delete_is_rejected_without_touching_history(task, monkeypatch):
|
|
sid, _, root, store = task
|
|
monkeypatch.setattr(task_deletion, "session_repo", task_inspection.session_repo)
|
|
monkeypatch.setattr(task_deletion, "store", store)
|
|
monkeypatch.setattr(task_deletion, "_check_runs", lambda _: None)
|
|
store.save_job({"session_id": sid, "status": "running", "position": 1})
|
|
app = FastAPI()
|
|
app.include_router(sessions.router)
|
|
client = TestClient(app)
|
|
assert client.get(f"/api/sessions/{sid}/deletion-preview").status_code == 409
|
|
assert client.post(f"/api/sessions/{sid}/delete").status_code == 409
|
|
assert (root / sid).is_dir()
|
|
assert store.job(sid)["status"] == "running"
|
|
|
|
|
|
def test_mcp_review_and_delete_require_user_intent_and_share_http_backend(monkeypatch):
|
|
request = AsyncMock(return_value={"test_result": {"source": "human"}})
|
|
monkeypatch.setattr(automation, "_request", request)
|
|
for action in ("review", "delete"):
|
|
with pytest.raises(ValueError, match="用户"):
|
|
task_manager.mobile_manage_task(action, "task")
|
|
request.assert_not_awaited()
|
|
result = task_manager.mobile_manage_task(
|
|
"review", "task", review_status="passed", review_reason="确认", user_confirmed=True
|
|
)
|
|
assert result["test_result"]["source"] == "human"
|
|
request.assert_awaited_once_with(
|
|
"PUT", "/api/automation/reports/task/review", json={"status": "passed", "reason": "确认"}
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_mcp_read_inspection_and_exploration_delete_routes(monkeypatch):
|
|
request = AsyncMock(return_value={"ok": True})
|
|
monkeypatch.setattr(automation, "_request", request)
|
|
monkeypatch.setattr(exploration, "_request", request)
|
|
await inspect_trace.mobile_inspect_trace("view_requests", "task", offset=20, limit=10)
|
|
request.assert_awaited_with(
|
|
"GET",
|
|
"/api/sessions/task/inspection",
|
|
params={"view": "requests", "offset": 20, "limit": 10},
|
|
)
|
|
with pytest.raises(ValueError, match="用户"):
|
|
await exploration.mobile_exploration("delete", exploration_id="e")
|
|
await exploration.mobile_exploration("delete", exploration_id="e", user_confirmed=True)
|
|
request.assert_awaited_with("DELETE", "/api/explorations/e")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_plugin_probe_only_targets_installed_plugin(monkeypatch):
|
|
async def request(method, path, **kwargs):
|
|
return {"plugins": [{"manifest": {"id": "demo"}}]} if path == "/api/plugins" else {}
|
|
|
|
mocked = AsyncMock(side_effect=request)
|
|
monkeypatch.setattr(automation, "_request", mocked)
|
|
result = await automation.mobile_workspace(
|
|
include_devices=False, plugin_id="demo", probe_plugin=True, include_development=True
|
|
)
|
|
assert result["plugins"][0]["manifest"]["id"] == "demo"
|
|
assert result["development"]["plugin_schema_version"] == 2
|
|
assert [call.args for call in mocked.call_args_list if call.args[0] == "POST"] == [
|
|
("POST", "/api/plugins/demo/diagnose")
|
|
]
|
|
|
|
|
|
def test_exploration_deletion_preview_keeps_validation_runs(monkeypatch):
|
|
repository = SimpleNamespace(
|
|
get=lambda _: {},
|
|
runs=lambda _: [
|
|
{"id": "explore-run", "kind": "exploration"},
|
|
{"id": "rerun", "kind": "validation"},
|
|
],
|
|
)
|
|
monkeypatch.setattr(task_deletion, "explorations", repository)
|
|
check = Mock()
|
|
monkeypatch.setattr(task_deletion, "_check_runs", check)
|
|
result = task_deletion.exploration_deletion_preview("e")
|
|
assert result["session_ids"] == ["explore-run"]
|
|
assert result["preserved_session_ids"] == ["rerun"]
|
|
check.assert_called_once()
|