feat: add official Codex skill and harden task execution
Install and update artemis-use with task diagnostics and plugin guidance. Preserve cross-device file receipts, review canonical reports, guard stage transitions and reduce small history summaries. Improve exploration lifecycle, request recovery and browser live connections.
This commit is contained in:
chenmingxuan committed
1 parent
2328a4df0c
commit
e1ea099592
128 files changed
+4944
-700
No files matched your search
@@ -0,0 +1,257 @@
|
||||
"""Read-only diagnostics and destructive-action gates against disposable task data."""
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
import pytest
|
||||
|
||||
from apps.admin_console.database.repositories.session_repository import SessionRepository
|
||||
from apps.admin_console.routers import sessions
|
||||
from apps.admin_console.services import report_export, task_deletion, task_inspection
|
||||
from apps.admin_console.services.automation_store import AutomationStore
|
||||
from apps.admin_console.services.request_diagnostics import request_diagnostics
|
||||
from artemis.data_engine.storage import StorageManager
|
||||
from artemis.runtime import trace_store
|
||||
from mcp_server.tools import automation, exploration, inspect_trace, task_manager
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def task(tmp_path, monkeypatch):
|
||||
sid = str(uuid4())
|
||||
db = tmp_path / "data_engine.db"
|
||||
StorageManager(db, tmp_path)
|
||||
store = AutomationStore(db)
|
||||
repo = SessionRepository(db)
|
||||
with store.connection() as connection:
|
||||
connection.execute(
|
||||
"INSERT INTO sessions (session_id, initial_goal, start_time, status, device_info) "
|
||||
"VALUES (?, 'observe', 1, 'completed', '{}')",
|
||||
(sid,),
|
||||
)
|
||||
connection.commit()
|
||||
monkeypatch.setattr(task_inspection, "session_repo", repo)
|
||||
monkeypatch.setattr(task_inspection, "store", store)
|
||||
monkeypatch.setattr(task_inspection.evidence_service, "task_root", lambda _: tmp_path / sid)
|
||||
monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path))
|
||||
monkeypatch.setattr(report_export, "store", store)
|
||||
monkeypatch.setattr(report_export, "TRACES_PATH", tmp_path)
|
||||
(tmp_path / sid).mkdir()
|
||||
return sid, db, tmp_path, store
|
||||
|
||||
|
||||
def add_trace(store, sid, kind, name, payload, timestamp=1):
|
||||
with store.connection() as connection:
|
||||
connection.execute(
|
||||
"INSERT INTO traces (trace_id,session_id,type,name,payload,timestamp) VALUES (?,?,?,?,?,?)",
|
||||
(str(uuid4()), sid, kind, name, json.dumps(payload), timestamp),
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
|
||||
def test_usage_uses_input_denominator_and_does_not_invent_missing_model(task):
|
||||
sid, _, _, store = task
|
||||
assert task_inspection.inspect_task(sid, "usage")["cache_hit_rate"] is None
|
||||
add_trace(
|
||||
store,
|
||||
sid,
|
||||
"llm_call",
|
||||
"llm_usage",
|
||||
{
|
||||
"node": "operator",
|
||||
"model": "model-a",
|
||||
"prompt_tokens": 1000,
|
||||
"cached_tokens": 900,
|
||||
"completion_tokens": 500,
|
||||
},
|
||||
)
|
||||
add_trace(
|
||||
store,
|
||||
sid,
|
||||
"llm_call",
|
||||
"llm_usage",
|
||||
{
|
||||
"node": "checker",
|
||||
"prompt_tokens": 1000,
|
||||
"cached_tokens": 0,
|
||||
"completion_tokens": 500,
|
||||
},
|
||||
)
|
||||
result = task_inspection.inspect_task(sid, "usage")
|
||||
assert result["cache_hit_rate"] == 0.45
|
||||
assert result["total_tokens"] == 3000
|
||||
assert result["by_node_model"][1]["model"] == "unknown"
|
||||
|
||||
|
||||
def test_log_tail_is_bounded_paginated_and_redacted(task, monkeypatch):
|
||||
sid, _, root, _ = task
|
||||
monkeypatch.setenv("TEST_API_KEY", "test-only-sensitive-value")
|
||||
lines = ["x" * 200 for _ in range(1000)] + [
|
||||
"old",
|
||||
"test-only-sensitive-value",
|
||||
"Bearer abcdef",
|
||||
"password=hush",
|
||||
]
|
||||
(root / sid / "stderr.log").write_text("\n".join(lines), "utf-8")
|
||||
result = task_inspection.inspect_task(sid, "logs", limit=2)
|
||||
error = result["logs"][1]
|
||||
assert error["older_bytes_omitted"] > 0
|
||||
assert error["next_offset"] == 2
|
||||
assert error["lines"] == ["Bearer [REDACTED]", "password=[REDACTED]"]
|
||||
older = task_inspection.inspect_task(sid, "logs", offset=2, limit=2)
|
||||
assert older["logs"][1]["lines"] == ["old", "[REDACTED]"]
|
||||
assert result["logs"][0]["available"] is False
|
||||
|
||||
|
||||
def test_report_pagination_preserves_total_and_caps_nested_content(task, monkeypatch):
|
||||
sid, _, _, _ = task
|
||||
monkeypatch.setattr(
|
||||
report_export,
|
||||
"session_report",
|
||||
lambda _: {
|
||||
"steps": [{"number": i} for i in range(35)],
|
||||
"content": "x" * 100000,
|
||||
"metadata": {"api_key": "hidden"},
|
||||
"nested": [[{"value": "x" * 100} for _ in range(1000)] for _ in range(1000)],
|
||||
},
|
||||
)
|
||||
result = task_inspection.inspect_task(sid, "report", offset=20, limit=10)
|
||||
assert result["steps"]["items"] == [{"number": i} for i in range(20, 30)]
|
||||
assert result["steps"]["total"] == 35
|
||||
assert result["steps"]["next_offset"] == 30
|
||||
assert result["metadata"]["api_key"] == "[REDACTED]"
|
||||
assert result["content_truncated"] is True
|
||||
assert len(json.dumps(result, ensure_ascii=False)) < 150000
|
||||
|
||||
|
||||
def test_request_pagination_reports_latest_phase_only(task):
|
||||
sid, db, _, store = task
|
||||
with store.connection() as connection:
|
||||
connection.execute("UPDATE sessions SET status='running' WHERE session_id=?", (sid,))
|
||||
connection.commit()
|
||||
add_trace(
|
||||
store, sid, "llm_diagnostic", "request", {"request_id": "r", "phase": "waiting_response"}, 1
|
||||
)
|
||||
add_trace(store, sid, "llm_diagnostic", "request", {"request_id": "r", "phase": "completed"}, 2)
|
||||
assert request_diagnostics(sid, db_path=db, limit=1)["next_offset"] == 1
|
||||
older = request_diagnostics(sid, db_path=db, limit=1, offset=1)
|
||||
assert older["requests"][0]["active"] is False
|
||||
assert older["next_offset"] is None
|
||||
|
||||
|
||||
def test_status_and_report_use_same_human_review_without_losing_ai_result(task):
|
||||
sid, db, root, store = task
|
||||
(root / sid / "run_outcome.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"test_result": {
|
||||
"status": "failed",
|
||||
"reason": "AI found mismatch",
|
||||
"evidence": ["step 1: actual differs from expected"],
|
||||
},
|
||||
}
|
||||
),
|
||||
"utf-8",
|
||||
)
|
||||
store.put(
|
||||
"case_review",
|
||||
sid,
|
||||
{"reviews": [{"status": "passed", "reason": "用户确认", "reviewed_at": 10}]},
|
||||
)
|
||||
result = task_manager.mobile_manage_task("status", sid)["test_result"]
|
||||
assert result == report_export.session_report(sid, db_path=db)["test_result"]
|
||||
assert result["source"] == "human"
|
||||
assert result["ai_result"]["status"] == "failed"
|
||||
|
||||
|
||||
def test_inspection_not_found_and_pagination_validation(task):
|
||||
app = FastAPI()
|
||||
app.include_router(sessions.router)
|
||||
client = TestClient(app)
|
||||
assert client.get(f"/api/sessions/{uuid4()}/inspection").status_code == 404
|
||||
assert client.get(f"/api/sessions/{task[0]}/inspection?limit=101").status_code == 422
|
||||
|
||||
|
||||
def test_active_normal_task_delete_is_rejected_without_touching_history(task, monkeypatch):
|
||||
sid, _, root, store = task
|
||||
monkeypatch.setattr(task_deletion, "session_repo", task_inspection.session_repo)
|
||||
monkeypatch.setattr(task_deletion, "store", store)
|
||||
monkeypatch.setattr(task_deletion, "_check_runs", lambda _: None)
|
||||
store.save_job({"session_id": sid, "status": "running", "position": 1})
|
||||
app = FastAPI()
|
||||
app.include_router(sessions.router)
|
||||
client = TestClient(app)
|
||||
assert client.get(f"/api/sessions/{sid}/deletion-preview").status_code == 409
|
||||
assert client.post(f"/api/sessions/{sid}/delete").status_code == 409
|
||||
assert (root / sid).is_dir()
|
||||
assert store.job(sid)["status"] == "running"
|
||||
|
||||
|
||||
def test_mcp_review_and_delete_require_user_intent_and_share_http_backend(monkeypatch):
|
||||
request = AsyncMock(return_value={"test_result": {"source": "human"}})
|
||||
monkeypatch.setattr(automation, "_request", request)
|
||||
for action in ("review", "delete"):
|
||||
with pytest.raises(ValueError, match="用户"):
|
||||
task_manager.mobile_manage_task(action, "task")
|
||||
request.assert_not_awaited()
|
||||
result = task_manager.mobile_manage_task(
|
||||
"review", "task", review_status="passed", review_reason="确认", user_confirmed=True
|
||||
)
|
||||
assert result["test_result"]["source"] == "human"
|
||||
request.assert_awaited_once_with(
|
||||
"PUT", "/api/automation/reports/task/review", json={"status": "passed", "reason": "确认"}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_read_inspection_and_exploration_delete_routes(monkeypatch):
|
||||
request = AsyncMock(return_value={"ok": True})
|
||||
monkeypatch.setattr(automation, "_request", request)
|
||||
monkeypatch.setattr(exploration, "_request", request)
|
||||
await inspect_trace.mobile_inspect_trace("view_requests", "task", offset=20, limit=10)
|
||||
request.assert_awaited_with(
|
||||
"GET",
|
||||
"/api/sessions/task/inspection",
|
||||
params={"view": "requests", "offset": 20, "limit": 10},
|
||||
)
|
||||
with pytest.raises(ValueError, match="用户"):
|
||||
await exploration.mobile_exploration("delete", exploration_id="e")
|
||||
await exploration.mobile_exploration("delete", exploration_id="e", user_confirmed=True)
|
||||
request.assert_awaited_with("DELETE", "/api/explorations/e")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plugin_probe_only_targets_installed_plugin(monkeypatch):
|
||||
async def request(method, path, **kwargs):
|
||||
return {"plugins": [{"manifest": {"id": "demo"}}]} if path == "/api/plugins" else {}
|
||||
|
||||
mocked = AsyncMock(side_effect=request)
|
||||
monkeypatch.setattr(automation, "_request", mocked)
|
||||
result = await automation.mobile_workspace(
|
||||
include_devices=False, plugin_id="demo", probe_plugin=True, include_development=True
|
||||
)
|
||||
assert result["plugins"][0]["manifest"]["id"] == "demo"
|
||||
assert result["development"]["plugin_schema_version"] == 2
|
||||
assert [call.args for call in mocked.call_args_list if call.args[0] == "POST"] == [
|
||||
("POST", "/api/plugins/demo/diagnose")
|
||||
]
|
||||
|
||||
|
||||
def test_exploration_deletion_preview_keeps_validation_runs(monkeypatch):
|
||||
repository = SimpleNamespace(
|
||||
get=lambda _: {},
|
||||
runs=lambda _: [
|
||||
{"id": "explore-run", "kind": "exploration"},
|
||||
{"id": "rerun", "kind": "validation"},
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(task_deletion, "explorations", repository)
|
||||
check = Mock()
|
||||
monkeypatch.setattr(task_deletion, "_check_runs", check)
|
||||
result = task_deletion.exploration_deletion_preview("e")
|
||||
assert result["session_ids"] == ["explore-run"]
|
||||
assert result["preserved_session_ids"] == ["rerun"]
|
||||
check.assert_called_once()
|
||||
Reference in new issue
Block a user