264 lines
10 KiB
Python
264 lines
10 KiB
Python
import json
|
|
import gzip
|
|
from types import SimpleNamespace
|
|
|
|
import httpx2
|
|
import pytest
|
|
|
|
from artemis.llm.responses_diagnostics import MAX_FRAME_BYTES, ResponsesObservation
|
|
from artemis.llm.diagnostics import ACTIVE_ATTEMPT, observe_response_headers
|
|
from artemis.llm.tool_constraints import MAX_REQUEST_BYTES, request_constraints, tool_constraints
|
|
|
|
|
|
def frame(kind, sequence, **fields):
|
|
return (
|
|
"data: "
|
|
+ json.dumps({"type": kind, "sequence_number": sequence, **fields}, ensure_ascii=False)
|
|
+ "\r\n\r\n"
|
|
).encode()
|
|
|
|
|
|
@pytest.mark.parametrize("size", [1, 37, 100000])
|
|
def test_raw_frames_split_unicode_and_done_snapshots_are_not_double_counted(size):
|
|
arguments = '{"text":"私人🙂"}'
|
|
data = b"".join(
|
|
[
|
|
frame("response.created", 0, response={"id": "resp_one"}),
|
|
frame(
|
|
"response.output_item.added",
|
|
1,
|
|
output_index=2,
|
|
item={
|
|
"type": "function_call",
|
|
"id": "fc_one",
|
|
"call_id": "call_one",
|
|
"name": "click",
|
|
"arguments": "",
|
|
},
|
|
),
|
|
frame(
|
|
"response.function_call_arguments.delta",
|
|
2,
|
|
output_index=2,
|
|
item_id="fc_one",
|
|
delta=arguments,
|
|
),
|
|
frame(
|
|
"response.function_call_arguments.done",
|
|
3,
|
|
output_index=2,
|
|
item_id="fc_one",
|
|
arguments=arguments,
|
|
),
|
|
frame(
|
|
"response.output_item.done",
|
|
4,
|
|
output_index=2,
|
|
item={"type": "function_call", "id": "fc_one", "arguments": arguments},
|
|
),
|
|
frame(
|
|
"response.completed",
|
|
5,
|
|
response={
|
|
"id": "resp_one",
|
|
"tools": [{"type": "function", "name": "click", "strict": True}],
|
|
},
|
|
),
|
|
]
|
|
)
|
|
observer = ResponsesObservation()
|
|
for offset in range(0, len(data), size):
|
|
observer.feed(data[offset : offset + size])
|
|
observer.end(eof=True)
|
|
stats = observer.snapshot()
|
|
assert stats["body_bytes"] == len(data)
|
|
assert stats["parse_errors"] == 0
|
|
assert stats["incomplete_sse_frame"] is False
|
|
(tool,) = stats["tools"]
|
|
assert tool["argument_chars"] == len(arguments)
|
|
assert (
|
|
tool["delta_events"]
|
|
== tool["arguments_done_events"]
|
|
== tool["output_item_done_events"]
|
|
== 1
|
|
)
|
|
assert tool["call_id"] == "call_one"
|
|
(response,) = stats["responses"]
|
|
assert response["sequence_first"] == 0 and response["sequence_last"] == 5
|
|
assert response["sequence_forward_gaps"] == response["sequence_non_increasing"] == 0
|
|
assert response["returned_tool_constraints"]["tools"][0]["strict"] == "true"
|
|
assert "私人" not in json.dumps(stats, ensure_ascii=False)
|
|
assert not observer.line and not observer.lines
|
|
|
|
|
|
def test_sequence_statistics_are_per_response_and_describe_observed_jumps():
|
|
observer = ResponsesObservation()
|
|
observer.event({"type": "response.in_progress", "sequence_number": 1})
|
|
observer.event({"type": "response.created", "sequence_number": 0, "response": {"id": "resp_a"}})
|
|
for seq in [2, 2, 1, 3, None]:
|
|
observer.event({"type": "response.in_progress", "sequence_number": seq})
|
|
observer.event({"type": "response.created", "sequence_number": 0, "response": {"id": "resp_b"}})
|
|
a, b = observer.snapshot()["responses"]
|
|
assert a["sequence_forward_gaps"] == a["sequence_forward_missing_total"] == 2
|
|
assert a["sequence_non_increasing"] == 2
|
|
assert a["sequence_equal_previous"] == 1
|
|
assert a["sequence_missing_events"] == 1
|
|
assert b["sequence_non_increasing"] == b["sequence_forward_gaps"] == 0
|
|
assert observer.data["unknown_response_events"] == 1
|
|
|
|
|
|
def test_oversized_frame_recovery_and_partial_eof_keep_memory_bounded():
|
|
observer = ResponsesObservation()
|
|
observer.feed(b"data: " + b"x" * MAX_FRAME_BYTES)
|
|
observer.feed(b"\r\n\r\n")
|
|
observer.feed(b"data: invalid\n\n")
|
|
observer.feed(frame("response.created", 0, response={"id": "resp_valid"}))
|
|
observer.feed(b'data: {"private":')
|
|
observer.end(eof=True)
|
|
stats = observer.snapshot()
|
|
assert stats["oversized_frames"] == 1
|
|
assert stats["parse_errors"] == 1
|
|
assert len(stats["responses"]) == 1
|
|
assert stats["incomplete_sse_frame"] is True
|
|
assert "private" not in repr(vars(observer))
|
|
assert not observer.line and not observer.lines
|
|
observer.end() # HTTP close can follow EOF; retain the original coverage gap.
|
|
assert observer.data["incomplete_sse_frame"] is True
|
|
|
|
|
|
def test_raw_tool_and_response_statistics_have_fixed_caps():
|
|
observer = ResponsesObservation()
|
|
observer.event({"type": "response.created", "response": {"id": "resp_one"}})
|
|
for index in range(100):
|
|
observer.event(
|
|
{
|
|
"type": "response.function_call_arguments.delta",
|
|
"item_id": f"fc_{index}",
|
|
"output_index": index,
|
|
"delta": " ",
|
|
}
|
|
)
|
|
for index in range(20):
|
|
observer.event({"type": "response.created", "response": {"id": f"resp_{index}"}})
|
|
assert len(observer.tools) == len(observer.fingerprints) == len(observer.aliases) == 64
|
|
assert len(observer.responses) == 16
|
|
assert observer.data["statistics_truncated"] is True
|
|
|
|
|
|
@pytest.mark.parametrize("size", [1, 37, MAX_FRAME_BYTES * 2])
|
|
def test_oversized_crlf_frame_recovers_across_arbitrary_read_boundaries(size):
|
|
observer = ResponsesObservation()
|
|
data = b"data: " + b"x" * MAX_FRAME_BYTES + b"\r\n\r\n"
|
|
data += frame("response.created", 0, response={"id": "resp_after_large"})
|
|
for offset in range(0, len(data), size):
|
|
observer.feed(data[offset : offset + size])
|
|
assert observer.data["oversized_frames"] == 1
|
|
assert observer.data["parse_errors"] == 0
|
|
assert list(observer.responses) == ["resp_after_large"]
|
|
assert not observer.line and not observer.lines and not observer.dropping
|
|
|
|
|
|
def test_alias_statistics_are_bounded_even_when_item_keeps_changing_index():
|
|
observer = ResponsesObservation()
|
|
observer.event({"response": {"id": "resp_one"}})
|
|
for index in range(1000):
|
|
observer.event(
|
|
{
|
|
"type": "response.function_call_arguments.delta",
|
|
"item_id": "fc_same",
|
|
"output_index": index,
|
|
"delta": " ",
|
|
}
|
|
)
|
|
assert len(observer.tools) == 1
|
|
assert len(observer.aliases) == 128
|
|
assert observer.data["statistics_truncated"] is True
|
|
|
|
|
|
def test_malformed_event_type_cannot_break_passive_observation():
|
|
observer = ResponsesObservation()
|
|
observer.feed(frame([], 0, response={"id": "resp_one"}))
|
|
assert observer.snapshot()["responses"][0]["events"] == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_compressed_body_is_explicitly_skipped_and_sdk_decoding_is_unchanged():
|
|
body = frame("response.created", 0, response={"id": "resp_one"})
|
|
compressed = gzip.compress(body)
|
|
|
|
class Stream(httpx2.AsyncByteStream):
|
|
async def __aiter__(self):
|
|
yield compressed
|
|
|
|
attempt = SimpleNamespace(
|
|
wire_streams=[],
|
|
data={},
|
|
headers=lambda _: None,
|
|
stream_work=lambda *_: None,
|
|
publish_if_due=lambda: None,
|
|
)
|
|
response = httpx2.Response(
|
|
200,
|
|
headers={"content-type": "text/event-stream", "content-encoding": "gzip"},
|
|
stream=Stream(),
|
|
)
|
|
token = ACTIVE_ATTEMPT.set(attempt)
|
|
try:
|
|
await observe_response_headers(response)
|
|
assert b"".join([chunk async for chunk in response.aiter_bytes()]) == body
|
|
stats = attempt.wire_streams[0].observation.snapshot()
|
|
assert stats["sse_observation"] == "skipped_content_encoding"
|
|
assert stats["body_bytes"] == len(compressed)
|
|
assert not stats["responses"] and not stats["parse_errors"]
|
|
finally:
|
|
await response.aclose()
|
|
ACTIVE_ATTEMPT.reset(token)
|
|
|
|
|
|
@pytest.mark.parametrize("strict,expected", [(None, "omitted"), (False, "false"), (True, "true")])
|
|
def test_actual_strict_state_and_optional_schema_are_observed_without_mutation(strict, expected):
|
|
tool = {
|
|
"type": "function",
|
|
"name": "click",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {"target": {"anyOf": [{"type": "string"}, {"type": "null"}]}},
|
|
"required": [],
|
|
},
|
|
}
|
|
if strict is not None:
|
|
tool["strict"] = strict
|
|
body = json.dumps(
|
|
{"tools": [tool], "input": "private prompt", "max_output_tokens": 2048}
|
|
).encode()
|
|
result = request_constraints(body)
|
|
assert result["max_output_tokens"] == 2048
|
|
(observed,) = result["tools"]
|
|
assert observed["strict"] == expected
|
|
assert observed["schema"]["optional_properties"] == observed["schema"]["open_objects"] == 1
|
|
assert "strict" not in tool if strict is None else tool["strict"] is strict
|
|
assert "private" not in json.dumps(result)
|
|
assert json.loads(body)["tools"][0] == tool
|
|
|
|
|
|
def test_constraints_bound_large_requests_namespaces_and_deep_schemas():
|
|
assert request_constraints(b"x" * (MAX_REQUEST_BYTES + 1)) == {"observation": "body_too_large"}
|
|
assert request_constraints(b"no JSON") == {"observation": "invalid_json"}
|
|
tools = [
|
|
{
|
|
"type": "namespace",
|
|
"name": "functions",
|
|
"tools": [{"type": "function", "name": "click"}],
|
|
},
|
|
{"type": "function", "name": "drag", "strict": False},
|
|
]
|
|
result = tool_constraints(tools)
|
|
assert {t["name"]: t["strict"] for t in result["tools"]} == {
|
|
"click": "omitted",
|
|
"drag": "false",
|
|
}
|
|
schema = {"type": "object", "properties": {str(i): {"type": "string"} for i in range(2000)}}
|
|
result = tool_constraints([{"type": "function", "name": "wide", "parameters": schema}])
|
|
assert result["tools"][0]["schema"]["truncated"] is True
|
|
assert result["tools"][0]["schema"]["nodes"] <= 1024
|