fix: improve cross-device evidence and exploration lifecycle
Align Responses cache affinity across execution and summary calls, preserve required checks and evidence uncertainty, and allow safe deletion of inactive explorations while retaining independent reruns. Validation: backend suite passed except intentional prompt golden drift, then all 17 transcript tests passed after snapshot refresh; 268 frontend tests and production build passed; Ruff, quality ratchet and core type checks passed.
This commit is contained in:
1 parent
f1bb5d02aa
commit
2328a4df0c
32 files changed
+748
-79
No files matched your search
@@ -93,6 +93,57 @@ async def test_summary_rules_and_goal_prefix_stay_identical_across_steps(mock_co
|
||||
assert "Step 2 Physical Action" in str(blocks2)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("duplicate", [True, False])
|
||||
async def test_summary_keeps_ui_once_without_losing_controller_errors(mock_context, duplicate):
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._llm = Mock(
|
||||
ainvoke=AsyncMock(return_value=AIMessage(content="我观察到当前设置,状态变化仍待确认。"))
|
||||
)
|
||||
ui = 'Window Brave\n[1] Text "未保存"\n[2] Button "重试"'
|
||||
outcome = "控制器错误: input rejected\n" + (ui if duplicate else "different earlier tree")
|
||||
outcome += "\n恢复状态: unknown"
|
||||
summarizer.dispatch(
|
||||
step_number=1,
|
||||
action_name="plug_click",
|
||||
action_args={},
|
||||
pre_img_bytes=b"before",
|
||||
post_img_bytes=b"after",
|
||||
exec_outcome=outcome,
|
||||
post_ui_elements_text=ui,
|
||||
observation_settled=False,
|
||||
)
|
||||
await summarizer.flush()
|
||||
_, blocks = _user_role_prompt(summarizer._llm.ainvoke.call_args.args[0])
|
||||
text = "\n".join(b.get("text", "") for b in blocks)
|
||||
assert text.count(ui) == 1
|
||||
assert "控制器错误: input rejected" in text and "恢复状态: unknown" in text
|
||||
assert "screen still changing" in text
|
||||
assert "AFTER ACTION UI ELEMENTS" in text
|
||||
if not duplicate:
|
||||
assert "different earlier tree" in text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_receipt_summary_retains_tree_even_when_duplicate_is_removed_for_model(mock_context):
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._llm = Mock(ainvoke=AsyncMock())
|
||||
ui = 'Window Brave\n[1] Text "未保存"'
|
||||
summarizer.dispatch(
|
||||
step_number=1,
|
||||
action_name="plug_get_window_state",
|
||||
action_args={},
|
||||
pre_img_bytes=b"same",
|
||||
post_img_bytes=b"same",
|
||||
exec_outcome="来源: PC\n" + ui,
|
||||
post_ui_elements_text=ui,
|
||||
receipt_only=True,
|
||||
)
|
||||
await summarizer.flush()
|
||||
summarizer._llm.ainvoke.assert_not_awaited()
|
||||
assert ui in summarizer.get_summary(1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarizer_dispatch_and_caching(mock_context):
|
||||
"""Verify non-blocking dispatch and summary generation."""
|
||||
|
||||
@@ -54,8 +54,8 @@ from artemis.memory.transcript import (
|
||||
|
||||
# SHA-256 snapshots of the legacy system message with the fixed inputs below.
|
||||
# Update these when an intentional template change alters the rendered prompt.
|
||||
GOLDEN_EMPTY_PLAN = "d715128625911aec46c83138e15aabfa967b385ab4c878509fe36ab4304fae2a"
|
||||
GOLDEN_SENTINEL_PLAN = "e2cf4ede75ffd9e283159f70dcd5c94a169c18e9f6dd4d1e7aacf460e956eaf3"
|
||||
GOLDEN_EMPTY_PLAN = "58bf034527b2d782f7746d8e379cb0decc44a05dfa18cb73153d49cd8265002b"
|
||||
GOLDEN_SENTINEL_PLAN = "b328052390a0a815c096b8ec4c5286421be487e7b6cd3bec55d6bc62a1f2518f"
|
||||
|
||||
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
|
||||
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
import re
|
||||
|
||||
from artemis.llm.cache_affinity import cache_affinity_options
|
||||
|
||||
|
||||
def context(session, exploration=None):
|
||||
return SimpleNamespace(
|
||||
data_engine=SimpleNamespace(current_session_id=session), exploration_id=exploration
|
||||
)
|
||||
|
||||
|
||||
def test_routing_identity_survives_resume_but_isolates_tasks_roles_and_models():
|
||||
ctx = context(uuid4(), "exploration")
|
||||
first = cache_affinity_options(ctx, "flash", "gpt-6-luna")
|
||||
ctx.data_engine.current_session_id = uuid4()
|
||||
assert first == cache_affinity_options(ctx, "flash", "gpt-6-luna")
|
||||
key = first["prompt_cache_key"]
|
||||
assert re.fullmatch(r"[-0-9A-Za-z._:]{1,128}", key)
|
||||
assert "exploration" not in key
|
||||
assert first["extra_headers"] == {"Thread-Id": key, "Session-Id": key}
|
||||
for role, model in [("checker", "gpt-6-luna"), ("flash", "gpt-6.1-sol")]:
|
||||
assert cache_affinity_options(ctx, role, model)["prompt_cache_key"] != key
|
||||
assert (
|
||||
cache_affinity_options(context("another"), "flash", "gpt-6-luna")["prompt_cache_key"] != key
|
||||
)
|
||||
|
||||
|
||||
def test_explicit_options_are_preserved_without_mutating_headers_or_adding_aliases():
|
||||
headers = {"thread_ID": "explicit-thread", "session-id": "explicit-session", "X-Test": "ok"}
|
||||
options = {"prompt_cache_key": "explicit.key", "extra_headers": headers, "temperature": 0}
|
||||
actual = cache_affinity_options(context("task"), "flash", "test", options)
|
||||
assert actual == options and actual is not options
|
||||
assert actual["extra_headers"] is not headers
|
||||
assert options["extra_headers"] == headers
|
||||
assert cache_affinity_options(None, "flash", "test") == {}
|
||||
for key in (None, "", "a" * 129, "unsafe\r\nheader"):
|
||||
assert cache_affinity_options(
|
||||
context("task"), "flash", "test", {"prompt_cache_key": key}
|
||||
) == {"prompt_cache_key": key}
|
||||
|
||||
|
||||
def test_concurrent_calls_do_not_share_or_mutate_task_headers():
|
||||
shared = {"extra_headers": {"X-Test": "shared"}}
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
results = list(
|
||||
pool.map(
|
||||
lambda task: cache_affinity_options(context(task), "flash", "test", shared),
|
||||
["task-a", "task-b"],
|
||||
)
|
||||
)
|
||||
assert results[0]["extra_headers"]["Thread-Id"] != results[1]["extra_headers"]["Thread-Id"]
|
||||
assert shared == {"extra_headers": {"X-Test": "shared"}}
|
||||
@@ -0,0 +1,166 @@
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
import pytest
|
||||
|
||||
from apps.admin_console.routers import sessions
|
||||
from apps.admin_console.services import exploration_deletion as deletion
|
||||
from apps.admin_console.services import report_export
|
||||
from apps.admin_console.services.automation_store import AutomationStore
|
||||
from artemis.config.exploration import ExplorationConfig
|
||||
from artemis.data_engine.storage import StorageManager
|
||||
from artemis.exploration import store as store_module
|
||||
from artemis.exploration.models import CaseDraft
|
||||
from artemis.exploration.store import ExplorationConflict, ExplorationStore
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def setup(tmp_path, monkeypatch):
|
||||
db = tmp_path / "data.db"
|
||||
traces = tmp_path / "traces"
|
||||
storage = StorageManager(db, traces)
|
||||
repository = ExplorationStore(AutomationStore(db))
|
||||
monkeypatch.setattr(store_module, "explorations", repository)
|
||||
monkeypatch.setattr(report_export, "TRACES_PATH", traces)
|
||||
monkeypatch.setattr(report_export, "EXPORT_ROOT", tmp_path / "exports")
|
||||
monkeypatch.setattr(deletion, "state", SimpleNamespace(queue_items=[], active_runs={}))
|
||||
monkeypatch.setattr(deletion, "read_state", lambda _: {})
|
||||
app = FastAPI()
|
||||
app.include_router(sessions.router)
|
||||
session = repository.create("探索页面", ExplorationConfig(), {}, "create")
|
||||
return repository, session["id"], storage, TestClient(app)
|
||||
|
||||
|
||||
def make_run(repository, sid, request, *, kind="exploration", finished=True):
|
||||
run = repository.reserve_run(
|
||||
sid,
|
||||
request,
|
||||
kind=kind,
|
||||
goal="观察",
|
||||
task_config={},
|
||||
case_version=1 if kind == "validation" else None,
|
||||
)
|
||||
if finished:
|
||||
repository.finish_run(run["id"], {"status": "cancelled"})
|
||||
return run["id"]
|
||||
|
||||
|
||||
def test_delete_conversation_removes_all_exploration_runs_but_keeps_reruns(setup):
|
||||
repository, sid, storage, client = setup
|
||||
first = make_run(repository, sid, "first")
|
||||
second = make_run(repository, sid, "continue")
|
||||
repository.save_case(sid, CaseDraft(name="可复用用例", goal="检查页面"), 0, "case")
|
||||
validation = make_run(repository, sid, "validate", kind="validation")
|
||||
unrelated = repository.create("另一探索", ExplorationConfig(), {}, "other")["id"]
|
||||
directories = []
|
||||
for run_id in (first, second, validation):
|
||||
repository.database.save_job({"session_id": run_id, "status": "cancelled", "position": 1})
|
||||
with repository.database.connection() as connection:
|
||||
connection.execute(
|
||||
"INSERT INTO sessions (session_id, initial_goal, start_time, status, device_info) VALUES (?, ?, 1, ?, '{}')",
|
||||
(run_id, "观察", "cancelled"),
|
||||
)
|
||||
connection.commit()
|
||||
for root in (storage.base_trace_dir, report_export.EXPORT_ROOT / "sessions"):
|
||||
folder = root / run_id
|
||||
folder.mkdir(parents=True)
|
||||
(folder / "evidence.txt").write_text("saved", encoding="utf-8")
|
||||
if run_id != validation:
|
||||
directories.append(folder)
|
||||
with repository._transaction() as connection:
|
||||
session = repository._require(connection, sid)
|
||||
session["status"] = "paused" # Resumable conversation, no running process.
|
||||
repository._save_session(connection, session)
|
||||
response = client.post(f"/api/sessions/{first}/delete")
|
||||
assert response.status_code == 200, response.text
|
||||
assert set(response.json()["deleted_session_ids"]) == {first, second}
|
||||
assert not any(folder.exists() for folder in directories)
|
||||
assert (storage.base_trace_dir / validation / "evidence.txt").exists()
|
||||
assert repository.database.job(validation) is not None
|
||||
assert repository.database.job(first) is None
|
||||
assert storage.get_session(UUID(validation)) is not None
|
||||
assert storage.get_session(UUID(first)) is None
|
||||
assert repository.get(unrelated)["goal"] == "另一探索"
|
||||
assert repository.run(validation) is None # Detach it from the deleted conversation.
|
||||
deleted_requests = [
|
||||
record
|
||||
for record in repository.database.records("exploration_request")
|
||||
if record.get("deleted")
|
||||
]
|
||||
assert len(deleted_requests) >= 5
|
||||
assert all("result" not in record for record in deleted_requests)
|
||||
with repository.database.connection() as connection:
|
||||
assert not connection.execute(
|
||||
"SELECT 1 FROM automation_records WHERE kind != 'exploration_request' "
|
||||
"AND json_extract(payload,'$.exploration_id')=?",
|
||||
(sid,),
|
||||
).fetchone()
|
||||
with pytest.raises(KeyError):
|
||||
repository.get(sid)
|
||||
with pytest.raises(ExplorationConflict, match="已删除"):
|
||||
repository.create("探索页面", ExplorationConfig(), {}, "create")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("state", ["running", "paused", "waiting_user", "demonstrating"])
|
||||
def test_live_execution_cannot_be_deleted_even_through_an_old_run(setup, state):
|
||||
repository, sid, _, client = setup
|
||||
first = make_run(repository, sid, "first")
|
||||
active = make_run(repository, sid, "continue", finished=False)
|
||||
repository.set_run_state(active, state)
|
||||
response = client.post(f"/api/sessions/{first}/delete")
|
||||
assert response.status_code == 409
|
||||
assert repository.get(sid)["active_run_id"] == active
|
||||
assert repository.run(first) is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["exploration_generation", "exploration_discussion"])
|
||||
def test_editorial_work_prevents_deletion(setup, kind):
|
||||
repository, sid, _, client = setup
|
||||
run_id = make_run(repository, sid, "first")
|
||||
repository.database.put(kind, "job", {"exploration_id": sid, "status": "running"})
|
||||
assert client.post(f"/api/sessions/{run_id}/delete").status_code == 409
|
||||
assert repository.get(sid)["status"] == "cancelled"
|
||||
|
||||
|
||||
def test_worker_still_exiting_prevents_deletion(setup, monkeypatch):
|
||||
repository, sid, _, client = setup
|
||||
run_id = make_run(repository, sid, "first")
|
||||
monkeypatch.setattr(
|
||||
deletion, "state", SimpleNamespace(queue_items=[], active_runs={run_id: {}})
|
||||
)
|
||||
assert client.post(f"/api/sessions/{run_id}/delete").status_code == 409
|
||||
assert repository.run(run_id) is not None
|
||||
|
||||
|
||||
def test_cleanup_failure_is_retryable_and_blocks_new_work(setup, monkeypatch):
|
||||
repository, sid, storage, client = setup
|
||||
run_id = make_run(repository, sid, "first")
|
||||
original = StorageManager.delete_session
|
||||
|
||||
def fail(*args):
|
||||
raise OSError("file busy")
|
||||
|
||||
monkeypatch.setattr(StorageManager, "delete_session", fail)
|
||||
response = client.post(f"/api/sessions/{run_id}/delete")
|
||||
assert response.status_code == 500
|
||||
with pytest.raises(ExplorationConflict, match="正在删除"):
|
||||
make_run(repository, sid, "restart")
|
||||
monkeypatch.setattr(StorageManager, "delete_session", original)
|
||||
assert client.post(f"/api/sessions/{run_id}/delete").status_code == 200
|
||||
assert not repository.list()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delayed_discussion_cannot_recreate_deleted_metadata(setup):
|
||||
from artemis.exploration.editorial import discuss
|
||||
|
||||
repository, sid, _, client = setup
|
||||
run_id = make_run(repository, sid, "first")
|
||||
message = repository.add_message(sid, "不再继续", "message")
|
||||
assert client.post(f"/api/sessions/{run_id}/delete").status_code == 200
|
||||
with pytest.raises(KeyError):
|
||||
await discuss(repository, sid, message)
|
||||
assert not repository.database.records("exploration_discussion")
|
||||
@@ -355,3 +355,46 @@ async def test_cache_key_is_stable_for_exploration_but_separates_roles(
|
||||
assert payloads[1]["prompt_cache_key"] != payloads[2]["prompt_cache_key"]
|
||||
assert "exploration-1" not in payloads[0]["prompt_cache_key"]
|
||||
assert payloads[1]["input"][:1] == payloads[0]["input"]
|
||||
for request, payload in zip(requests, payloads):
|
||||
assert request.headers["thread-id"] == payload["prompt_cache_key"]
|
||||
assert request.headers["session-id"] == payload["prompt_cache_key"]
|
||||
assert request.headers["x-client-request-id"]
|
||||
assert "extra_headers" not in payload
|
||||
assert requests[0].headers["x-client-request-id"] != requests[1].headers["x-client-request-id"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auxiliary_summary_calls_send_role_scoped_affinity_on_responses(transport_model):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from artemis.agents.flash.summarizer import VisualStepSummarizer
|
||||
from artemis.memory.chunking import StepCapsuleLens
|
||||
|
||||
model, requests, outputs = transport_model
|
||||
outputs.extend(
|
||||
[text_output("我看到页面显示当前设置,尚未确认其他状态变化。") for _ in range(4)]
|
||||
)
|
||||
ctx = SimpleNamespace(data_engine=SimpleNamespace(current_session_id="summary-task"))
|
||||
lens = StepCapsuleLens(model_name="test-model", llm=model, ctx=ctx)
|
||||
for text in ("first chunk", "next chunk"):
|
||||
await lens._invoke(model, [HumanMessage(content=text)], "test-model")
|
||||
visual = VisualStepSummarizer(ctx, model_name="test-model")
|
||||
visual._llm = model
|
||||
for number in (1, 2):
|
||||
visual._step_inputs[number] = {
|
||||
"step_number": number,
|
||||
"action_name": "press_key",
|
||||
"action_args": {},
|
||||
"pre_img_bytes": None,
|
||||
"post_img_bytes": None,
|
||||
"exec_outcome": "dispatched",
|
||||
}
|
||||
assert await visual._attempt(number)
|
||||
keys = []
|
||||
for request in requests:
|
||||
payload = json.loads(request.content)
|
||||
assert request.url.path == "/v1/responses"
|
||||
key = payload["prompt_cache_key"]
|
||||
assert request.headers["thread-id"] == request.headers["session-id"] == key
|
||||
keys.append(key)
|
||||
assert len(keys) == 4 and keys[0] == keys[1] and keys[2] == keys[3] and keys[0] != keys[2]
|
||||
@@ -13,6 +13,7 @@ from artemis.llm.diagnostics import (
|
||||
RequestAttempt,
|
||||
error_text,
|
||||
input_images,
|
||||
observe_request_headers,
|
||||
observe_response_headers,
|
||||
)
|
||||
from artemis.llm.router import ModelEndpoint
|
||||
@@ -34,6 +35,36 @@ class Engine:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_hook_correlates_api_calls_without_logging_header_values():
|
||||
engine = Engine()
|
||||
attempt = RequestAttempt(
|
||||
engine, request_id="local-request", attempt=1, timeout=300, max_retries=1, metadata={}
|
||||
)
|
||||
token = ACTIVE_ATTEMPT.set(attempt)
|
||||
try:
|
||||
request = httpx.Request(
|
||||
"POST",
|
||||
"https://example.test/v1/responses",
|
||||
headers={
|
||||
"Thread-Id": "private-thread",
|
||||
"Session-Id": "private-session",
|
||||
"Authorization": "Bearer private-key",
|
||||
},
|
||||
)
|
||||
await observe_request_headers(request)
|
||||
assert request.headers["x-client-request-id"] == "local-request"
|
||||
assert attempt.data["client_request_id_matches"] is True
|
||||
assert len(attempt.data["routing_header_fingerprints"]) == 2
|
||||
assert "private" not in json.dumps(engine.records, default=str)
|
||||
request.headers["X-Client-Request-Id"] = "caller-id"
|
||||
await observe_request_headers(request)
|
||||
assert request.headers["x-client-request-id"] == "caller-id"
|
||||
assert attempt.data["client_request_id_matches"] is False
|
||||
finally:
|
||||
ACTIVE_ATTEMPT.reset(token)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_headers_without_body_timeout_then_retry_keeps_evidence(monkeypatch):
|
||||
engine = Engine()
|
||||
|
||||
Reference in new issue
Block a user