fix: improve cross-device evidence and exploration lifecycle

Align Responses cache affinity across execution and summary calls, preserve required checks and evidence uncertainty, and allow safe deletion of inactive explorations while retaining independent reruns.

Validation: backend suite passed except intentional prompt golden drift, then all 17 transcript tests passed after snapshot refresh; 268 frontend tests and production build passed; Ruff, quality ratchet and core type checks passed.
This commit is contained in:
chenmingxuan committed 2026-10-17 19:43:20 +08:00
1 parent f1bb5d02aa
commit 2328a4df0c
32 files changed
+748 -79

No files matched your search

@@ -93,6 +93,57 @@ async def test_summary_rules_and_goal_prefix_stay_identical_across_steps(mock_co
assert "Step 2 Physical Action" in str(blocks2)
@pytest.mark.asyncio
@pytest.mark.parametrize("duplicate", [True, False])
async def test_summary_keeps_ui_once_without_losing_controller_errors(mock_context, duplicate):
summarizer = VisualStepSummarizer(mock_context)
summarizer._llm = Mock(
ainvoke=AsyncMock(return_value=AIMessage(content="我观察到当前设置,状态变化仍待确认。"))
)
ui = 'Window Brave\n[1] Text "未保存"\n[2] Button "重试"'
outcome = "控制器错误: input rejected\n" + (ui if duplicate else "different earlier tree")
outcome += "\n恢复状态: unknown"
summarizer.dispatch(
step_number=1,
action_name="plug_click",
action_args={},
pre_img_bytes=b"before",
post_img_bytes=b"after",
exec_outcome=outcome,
post_ui_elements_text=ui,
observation_settled=False,
)
await summarizer.flush()
_, blocks = _user_role_prompt(summarizer._llm.ainvoke.call_args.args[0])
text = "\n".join(b.get("text", "") for b in blocks)
assert text.count(ui) == 1
assert "控制器错误: input rejected" in text and "恢复状态: unknown" in text
assert "screen still changing" in text
assert "AFTER ACTION UI ELEMENTS" in text
if not duplicate:
assert "different earlier tree" in text
@pytest.mark.asyncio
async def test_receipt_summary_retains_tree_even_when_duplicate_is_removed_for_model(mock_context):
summarizer = VisualStepSummarizer(mock_context)
summarizer._llm = Mock(ainvoke=AsyncMock())
ui = 'Window Brave\n[1] Text "未保存"'
summarizer.dispatch(
step_number=1,
action_name="plug_get_window_state",
action_args={},
pre_img_bytes=b"same",
post_img_bytes=b"same",
exec_outcome="来源: PC\n" + ui,
post_ui_elements_text=ui,
receipt_only=True,
)
await summarizer.flush()
summarizer._llm.ainvoke.assert_not_awaited()
assert ui in summarizer.get_summary(1)
@pytest.mark.asyncio
async def test_summarizer_dispatch_and_caching(mock_context):
"""Verify non-blocking dispatch and summary generation."""
@@ -54,8 +54,8 @@ from artemis.memory.transcript import (
# SHA-256 snapshots of the legacy system message with the fixed inputs below.
# Update these when an intentional template change alters the rendered prompt.
GOLDEN_EMPTY_PLAN = "d715128625911aec46c83138e15aabfa967b385ab4c878509fe36ab4304fae2a"
GOLDEN_SENTINEL_PLAN = "e2cf4ede75ffd9e283159f70dcd5c94a169c18e9f6dd4d1e7aacf460e956eaf3"
GOLDEN_EMPTY_PLAN = "58bf034527b2d782f7746d8e379cb0decc44a05dfa18cb73153d49cd8265002b"
GOLDEN_SENTINEL_PLAN = "b328052390a0a815c096b8ec4c5286421be487e7b6cd3bec55d6bc62a1f2518f"
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
+55
View File
@@ -0,0 +1,55 @@
from concurrent.futures import ThreadPoolExecutor
from types import SimpleNamespace
from uuid import uuid4
import re
from artemis.llm.cache_affinity import cache_affinity_options
def context(session, exploration=None):
return SimpleNamespace(
data_engine=SimpleNamespace(current_session_id=session), exploration_id=exploration
)
def test_routing_identity_survives_resume_but_isolates_tasks_roles_and_models():
ctx = context(uuid4(), "exploration")
first = cache_affinity_options(ctx, "flash", "gpt-6-luna")
ctx.data_engine.current_session_id = uuid4()
assert first == cache_affinity_options(ctx, "flash", "gpt-6-luna")
key = first["prompt_cache_key"]
assert re.fullmatch(r"[-0-9A-Za-z._:]{1,128}", key)
assert "exploration" not in key
assert first["extra_headers"] == {"Thread-Id": key, "Session-Id": key}
for role, model in [("checker", "gpt-6-luna"), ("flash", "gpt-6.1-sol")]:
assert cache_affinity_options(ctx, role, model)["prompt_cache_key"] != key
assert (
cache_affinity_options(context("another"), "flash", "gpt-6-luna")["prompt_cache_key"] != key
)
def test_explicit_options_are_preserved_without_mutating_headers_or_adding_aliases():
headers = {"thread_ID": "explicit-thread", "session-id": "explicit-session", "X-Test": "ok"}
options = {"prompt_cache_key": "explicit.key", "extra_headers": headers, "temperature": 0}
actual = cache_affinity_options(context("task"), "flash", "test", options)
assert actual == options and actual is not options
assert actual["extra_headers"] is not headers
assert options["extra_headers"] == headers
assert cache_affinity_options(None, "flash", "test") == {}
for key in (None, "", "a" * 129, "unsafe\r\nheader"):
assert cache_affinity_options(
context("task"), "flash", "test", {"prompt_cache_key": key}
) == {"prompt_cache_key": key}
def test_concurrent_calls_do_not_share_or_mutate_task_headers():
shared = {"extra_headers": {"X-Test": "shared"}}
with ThreadPoolExecutor(max_workers=2) as pool:
results = list(
pool.map(
lambda task: cache_affinity_options(context(task), "flash", "test", shared),
["task-a", "task-b"],
)
)
assert results[0]["extra_headers"]["Thread-Id"] != results[1]["extra_headers"]["Thread-Id"]
assert shared == {"extra_headers": {"X-Test": "shared"}}
+166
View File
@@ -0,0 +1,166 @@
import json
from types import SimpleNamespace
from uuid import UUID
from fastapi import FastAPI
from fastapi.testclient import TestClient
import pytest
from apps.admin_console.routers import sessions
from apps.admin_console.services import exploration_deletion as deletion
from apps.admin_console.services import report_export
from apps.admin_console.services.automation_store import AutomationStore
from artemis.config.exploration import ExplorationConfig
from artemis.data_engine.storage import StorageManager
from artemis.exploration import store as store_module
from artemis.exploration.models import CaseDraft
from artemis.exploration.store import ExplorationConflict, ExplorationStore
@pytest.fixture
def setup(tmp_path, monkeypatch):
db = tmp_path / "data.db"
traces = tmp_path / "traces"
storage = StorageManager(db, traces)
repository = ExplorationStore(AutomationStore(db))
monkeypatch.setattr(store_module, "explorations", repository)
monkeypatch.setattr(report_export, "TRACES_PATH", traces)
monkeypatch.setattr(report_export, "EXPORT_ROOT", tmp_path / "exports")
monkeypatch.setattr(deletion, "state", SimpleNamespace(queue_items=[], active_runs={}))
monkeypatch.setattr(deletion, "read_state", lambda _: {})
app = FastAPI()
app.include_router(sessions.router)
session = repository.create("探索页面", ExplorationConfig(), {}, "create")
return repository, session["id"], storage, TestClient(app)
def make_run(repository, sid, request, *, kind="exploration", finished=True):
run = repository.reserve_run(
sid,
request,
kind=kind,
goal="观察",
task_config={},
case_version=1 if kind == "validation" else None,
)
if finished:
repository.finish_run(run["id"], {"status": "cancelled"})
return run["id"]
def test_delete_conversation_removes_all_exploration_runs_but_keeps_reruns(setup):
repository, sid, storage, client = setup
first = make_run(repository, sid, "first")
second = make_run(repository, sid, "continue")
repository.save_case(sid, CaseDraft(name="可复用用例", goal="检查页面"), 0, "case")
validation = make_run(repository, sid, "validate", kind="validation")
unrelated = repository.create("另一探索", ExplorationConfig(), {}, "other")["id"]
directories = []
for run_id in (first, second, validation):
repository.database.save_job({"session_id": run_id, "status": "cancelled", "position": 1})
with repository.database.connection() as connection:
connection.execute(
"INSERT INTO sessions (session_id, initial_goal, start_time, status, device_info) VALUES (?, ?, 1, ?, '{}')",
(run_id, "观察", "cancelled"),
)
connection.commit()
for root in (storage.base_trace_dir, report_export.EXPORT_ROOT / "sessions"):
folder = root / run_id
folder.mkdir(parents=True)
(folder / "evidence.txt").write_text("saved", encoding="utf-8")
if run_id != validation:
directories.append(folder)
with repository._transaction() as connection:
session = repository._require(connection, sid)
session["status"] = "paused" # Resumable conversation, no running process.
repository._save_session(connection, session)
response = client.post(f"/api/sessions/{first}/delete")
assert response.status_code == 200, response.text
assert set(response.json()["deleted_session_ids"]) == {first, second}
assert not any(folder.exists() for folder in directories)
assert (storage.base_trace_dir / validation / "evidence.txt").exists()
assert repository.database.job(validation) is not None
assert repository.database.job(first) is None
assert storage.get_session(UUID(validation)) is not None
assert storage.get_session(UUID(first)) is None
assert repository.get(unrelated)["goal"] == "另一探索"
assert repository.run(validation) is None # Detach it from the deleted conversation.
deleted_requests = [
record
for record in repository.database.records("exploration_request")
if record.get("deleted")
]
assert len(deleted_requests) >= 5
assert all("result" not in record for record in deleted_requests)
with repository.database.connection() as connection:
assert not connection.execute(
"SELECT 1 FROM automation_records WHERE kind != 'exploration_request' "
"AND json_extract(payload,'$.exploration_id')=?",
(sid,),
).fetchone()
with pytest.raises(KeyError):
repository.get(sid)
with pytest.raises(ExplorationConflict, match="已删除"):
repository.create("探索页面", ExplorationConfig(), {}, "create")
@pytest.mark.parametrize("state", ["running", "paused", "waiting_user", "demonstrating"])
def test_live_execution_cannot_be_deleted_even_through_an_old_run(setup, state):
repository, sid, _, client = setup
first = make_run(repository, sid, "first")
active = make_run(repository, sid, "continue", finished=False)
repository.set_run_state(active, state)
response = client.post(f"/api/sessions/{first}/delete")
assert response.status_code == 409
assert repository.get(sid)["active_run_id"] == active
assert repository.run(first) is not None
@pytest.mark.parametrize("kind", ["exploration_generation", "exploration_discussion"])
def test_editorial_work_prevents_deletion(setup, kind):
repository, sid, _, client = setup
run_id = make_run(repository, sid, "first")
repository.database.put(kind, "job", {"exploration_id": sid, "status": "running"})
assert client.post(f"/api/sessions/{run_id}/delete").status_code == 409
assert repository.get(sid)["status"] == "cancelled"
def test_worker_still_exiting_prevents_deletion(setup, monkeypatch):
repository, sid, _, client = setup
run_id = make_run(repository, sid, "first")
monkeypatch.setattr(
deletion, "state", SimpleNamespace(queue_items=[], active_runs={run_id: {}})
)
assert client.post(f"/api/sessions/{run_id}/delete").status_code == 409
assert repository.run(run_id) is not None
def test_cleanup_failure_is_retryable_and_blocks_new_work(setup, monkeypatch):
repository, sid, storage, client = setup
run_id = make_run(repository, sid, "first")
original = StorageManager.delete_session
def fail(*args):
raise OSError("file busy")
monkeypatch.setattr(StorageManager, "delete_session", fail)
response = client.post(f"/api/sessions/{run_id}/delete")
assert response.status_code == 500
with pytest.raises(ExplorationConflict, match="正在删除"):
make_run(repository, sid, "restart")
monkeypatch.setattr(StorageManager, "delete_session", original)
assert client.post(f"/api/sessions/{run_id}/delete").status_code == 200
assert not repository.list()
@pytest.mark.asyncio
async def test_delayed_discussion_cannot_recreate_deleted_metadata(setup):
from artemis.exploration.editorial import discuss
repository, sid, _, client = setup
run_id = make_run(repository, sid, "first")
message = repository.add_message(sid, "不再继续", "message")
assert client.post(f"/api/sessions/{run_id}/delete").status_code == 200
with pytest.raises(KeyError):
await discuss(repository, sid, message)
assert not repository.database.records("exploration_discussion")
+43
View File
@@ -355,3 +355,46 @@ async def test_cache_key_is_stable_for_exploration_but_separates_roles(
assert payloads[1]["prompt_cache_key"] != payloads[2]["prompt_cache_key"]
assert "exploration-1" not in payloads[0]["prompt_cache_key"]
assert payloads[1]["input"][:1] == payloads[0]["input"]
for request, payload in zip(requests, payloads):
assert request.headers["thread-id"] == payload["prompt_cache_key"]
assert request.headers["session-id"] == payload["prompt_cache_key"]
assert request.headers["x-client-request-id"]
assert "extra_headers" not in payload
assert requests[0].headers["x-client-request-id"] != requests[1].headers["x-client-request-id"]
@pytest.mark.asyncio
async def test_auxiliary_summary_calls_send_role_scoped_affinity_on_responses(transport_model):
from types import SimpleNamespace
from artemis.agents.flash.summarizer import VisualStepSummarizer
from artemis.memory.chunking import StepCapsuleLens
model, requests, outputs = transport_model
outputs.extend(
[text_output("我看到页面显示当前设置,尚未确认其他状态变化。") for _ in range(4)]
)
ctx = SimpleNamespace(data_engine=SimpleNamespace(current_session_id="summary-task"))
lens = StepCapsuleLens(model_name="test-model", llm=model, ctx=ctx)
for text in ("first chunk", "next chunk"):
await lens._invoke(model, [HumanMessage(content=text)], "test-model")
visual = VisualStepSummarizer(ctx, model_name="test-model")
visual._llm = model
for number in (1, 2):
visual._step_inputs[number] = {
"step_number": number,
"action_name": "press_key",
"action_args": {},
"pre_img_bytes": None,
"post_img_bytes": None,
"exec_outcome": "dispatched",
}
assert await visual._attempt(number)
keys = []
for request in requests:
payload = json.loads(request.content)
assert request.url.path == "/v1/responses"
key = payload["prompt_cache_key"]
assert request.headers["thread-id"] == request.headers["session-id"] == key
keys.append(key)
assert len(keys) == 4 and keys[0] == keys[1] and keys[2] == keys[3] and keys[0] != keys[2]
+31
View File
@@ -13,6 +13,7 @@ from artemis.llm.diagnostics import (
RequestAttempt,
error_text,
input_images,
observe_request_headers,
observe_response_headers,
)
from artemis.llm.router import ModelEndpoint
@@ -34,6 +35,36 @@ class Engine:
pass
@pytest.mark.asyncio
async def test_request_hook_correlates_api_calls_without_logging_header_values():
engine = Engine()
attempt = RequestAttempt(
engine, request_id="local-request", attempt=1, timeout=300, max_retries=1, metadata={}
)
token = ACTIVE_ATTEMPT.set(attempt)
try:
request = httpx.Request(
"POST",
"https://example.test/v1/responses",
headers={
"Thread-Id": "private-thread",
"Session-Id": "private-session",
"Authorization": "Bearer private-key",
},
)
await observe_request_headers(request)
assert request.headers["x-client-request-id"] == "local-request"
assert attempt.data["client_request_id_matches"] is True
assert len(attempt.data["routing_header_fingerprints"]) == 2
assert "private" not in json.dumps(engine.records, default=str)
request.headers["X-Client-Request-Id"] = "caller-id"
await observe_request_headers(request)
assert request.headers["x-client-request-id"] == "caller-id"
assert attempt.data["client_request_id_matches"] is False
finally:
ACTIVE_ATTEMPT.reset(token)
@pytest.mark.asyncio
async def test_headers_without_body_timeout_then_retry_keeps_evidence(monkeypatch):
engine = Engine()