feat: add unified 1.0.0 skill collection
This commit is contained in:
chenmingxuan committed
commit
cc4c380583
87 files changed
+6963
No files matched your search
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"name": "eapil-llmapi-structured",
|
||||
"version": "1.0.0",
|
||||
"description": "易瞳 LLM API 通用调用指南：选择、构建和实测 Chat、Responses、结构化输出、工具、视觉、流式、Embedding、Rerank 与 Decisions 请求，核对模型和中转渠道的实际兼容性。",
|
||||
"author": {
|
||||
"name": "EAPIL",
|
||||
"url": "https://git.playones.com/arechen/EapilSkillMarket"
|
||||
},
|
||||
"homepage": "https://git.playones.com/arechen/EapilSkillMarket",
|
||||
"repository": "https://git.playones.com/arechen/EapilSkillMarket",
|
||||
"license": "Proprietary",
|
||||
"keywords": [
|
||||
"eapil",
|
||||
"codex-skill",
|
||||
"eapil-llmapi-structured"
|
||||
],
|
||||
"skills": "./skills/",
|
||||
"interface": {
|
||||
"displayName": "易瞳LLM调用指南",
|
||||
"shortDescription": "Chat、Responses、结构化输出、Embedding、Rerank 和 Decisions 的调用与实测指南。",
|
||||
"longDescription": "按任务选择易瞳 LLM API 调用模式，提供端点参数、工具与视觉输入、流式解析、结构化校验、检索和决策示例。配套单请求探测脚本与有日期的实测记录，明确模型和渠道差异；保留原插件 ID 兼容已有引用。",
|
||||
"developerName": "EAPIL",
|
||||
"category": "Developer Tools",
|
||||
"capabilities": [
|
||||
"Read",
|
||||
"Write"
|
||||
],
|
||||
"defaultPrompt": [
|
||||
"使用 易瞳LLM调用指南 帮我选择端点、构建请求并验证结果。"
|
||||
],
|
||||
"websiteURL": "https://git.playones.com/arechen/EapilSkillMarket",
|
||||
"privacyPolicyURL": "https://git.playones.com/arechen/EapilSkillMarket",
|
||||
"termsOfServiceURL": "https://git.playones.com/arechen/EapilSkillMarket",
|
||||
"brandColor": "#2563EB",
|
||||
"screenshots": [],
|
||||
"composerIcon": "./assets/market-icon.svg",
|
||||
"logo": "./assets/market-icon.svg"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="360" height="360" viewBox="0 0 32 32"><title>易瞳LLM结构化调用</title><g fill="none"><path fill="#d9d9d9" d="M9.472 28.531L17.68 3H21l.006.016A4 4 0 0 1 24.117 5l5.485 9.5a3 3 0 0 1 0 3l-5.773 10A3 3 0 0 1 21.23 29H11l-.004-.016a4 4 0 0 1-1.524-.453"/><path fill="url(#SVGtIZlwdLy)" d="M9.472 28.531L17.68 3H21l.006.016A4 4 0 0 1 24.117 5l5.485 9.5a3 3 0 0 1 0 3l-5.773 10A3 3 0 0 1 21.23 29H11l-.004-.016a4 4 0 0 1-1.524-.453"/><path fill="url(#SVGS9366cPJ)" fill-opacity=".5" d="M9.472 28.531L17.68 3H21l.006.016A4 4 0 0 1 24.117 5l5.485 9.5a3 3 0 0 1 0 3l-5.773 10A3 3 0 0 1 21.23 29H11l-.004-.016a4 4 0 0 1-1.524-.453"/><path fill="url(#SVGLyzYJd2y)" d="m14 26.5l4-21l.043-.01A3 3 0 0 1 21 3H10.774a3 3 0 0 0-2.599 1.5l-5.773 10a3 3 0 0 0 0 3L7.887 27c.285.494.667.913 1.114 1.237a3 3 0 0 0 4.952-1.706z"/><path fill="url(#SVGTHQSMcNS)" fill-opacity=".4" d="m14 26.5l4-21l.043-.01A3 3 0 0 1 21 3H10.774a3 3 0 0 0-2.599 1.5l-5.773 10a3 3 0 0 0 0 3L7.887 27c.285.494.667.913 1.114 1.237a3 3 0 0 0 4.952-1.706z"/><defs><radialGradient id="SVGtIZlwdLy" cx="0" cy="0" r="1" gradientTransform="rotate(-87.021 26.23 10.402)scale(37.937 42.6996)" gradientUnits="userSpaceOnUse"><stop stop-color="#ffc470"/><stop offset=".251" stop-color="#ff835c"/><stop offset=".584" stop-color="#f24a9d"/><stop offset=".871" stop-color="#b339f0"/><stop offset="1" stop-color="#c354ff"/></radialGradient><radialGradient id="SVGS9366cPJ" cx="0" cy="0" r="1" gradientTransform="rotate(214.973 10.84 10.875)scale(27.8693 26.4973)" gradientUnits="userSpaceOnUse"><stop offset=".709" stop-color="#ffb357" stop-opacity="0"/><stop offset=".942" stop-color="#ffb357"/></radialGradient><radialGradient id="SVGLyzYJd2y" cx="0" cy="0" r="1" gradientTransform="rotate(201.076 16.049 10.671)scale(35.1186 31.3225)" gradientUnits="userSpaceOnUse"><stop offset=".222" stop-color="#4e46e2"/><stop offset=".578" stop-color="#625df6"/><stop offset=".955" stop-color="#e37dff"/></radialGradient><linearGradient id="SVGTHQSMcNS" x1="7.792" x2="16.073" y1="13.771" y2="15.597" gradientUnits="userSpaceOnUse"><stop stop-color="#7563f7" stop-opacity="0"/><stop offset=".986" stop-color="#4916ae"/></linearGradient></defs></g></svg>
|
||||
|
After Width: | Height: | Size: 2.2 KiB |
@@ -0,0 +1,41 @@
|
||||
---
|
||||
name: eapil-llmapi-structured
|
||||
description: Build, test, or debug EAPIL LLM API requests across Chat, Responses, structured output, tools, streaming, embeddings, reranking, and typed Decisions.
|
||||
---
|
||||
|
||||
# EAPIL LLM API guide
|
||||
|
||||
Choose the protocol for the task and model, then verify the deployed gateway's behavior. The historical skill ID remains for compatibility; this guide covers general LLM API calls, not only structured JSON.
|
||||
|
||||
## Choose the mode
|
||||
|
||||
| Need | EAPIL endpoint | Read when relevant |
|
||||
|---|---|---|
|
||||
| Conversation, compatible clients, vision or client tools | `/v1/chat/completions` | [Chat and Responses](references/chat-responses.md) |
|
||||
| Responses-native input/output, tools or streaming | `/v1/responses` | [Chat and Responses](references/chat-responses.md) |
|
||||
| JSON conforming to a schema | Either text endpoint, if the model/channel supports it | [Structured output](references/structured-output.md) |
|
||||
| Text vectors for retrieval or similarity | `/v1/embeddings` | [Embedding and Rerank](references/retrieval.md) |
|
||||
| Reorder candidate documents against a query | `/v1/rerank` | [Embedding and Rerank](references/retrieval.md) |
|
||||
| Probabilistic yes/no, categorical choice or ordinal scoring | `/v1/alpha/decisions`, alias `/api/alpha/decisions` | [Decisions](references/decisions.md) |
|
||||
|
||||
Chat is a first-class mode, not a universal fallback. Responses is not inherently cheaper or faster. Decisions uses its own `state`/`questions` protocol; the inspected gateway does not automatically convert Chat/Responses requests into Decisions. Image and voice workflows have their own repository skills.
|
||||
|
||||
## Use the actual integration
|
||||
|
||||
Default base URL is `https://api-n-cd.playones.com/v1`. Respect the user's intended deployment and available model; do not force a model/version from an old example. Query `/v1/models` when availability matters. A listed ID does not prove each endpoint, parameter, tool, schema or modality is supported. Model aliases and channel conversion can differ from the public provider's defaults.
|
||||
|
||||
Use `EAPIL_API_KEY`, then `OPENAI_API_KEY`, or the user's specified environment source. Keep credentials, private inputs, vectors and raw provider responses out of committed artifacts and routine logs. Use synthetic small probes for compatibility; do not replay production data or fan out broad paid benchmarks merely to verify an example.
|
||||
|
||||
Read only the relevant mode reference. Keep endpoint-specific field shapes distinct: Responses `input` and `text.format`, Chat `messages` and `response_format`, Decisions `state` and `questions`. Pass explicit supported values without dropping `0`, `false` or empty collections that carry meaning. Omit optional model-specific parameters when their support is unknown.
|
||||
|
||||
## Test the contract
|
||||
|
||||
Use [smoke_api.py](scripts/smoke_api.py) with an explicit UTF-8 JSON payload for each mode. It supports dry runs and Chat/Responses SSE, omits generated text and vectors from summaries, and performs mode-specific structural checks. Decisions summaries include labels/answers; dry runs print the payload. It makes one request and executes no returned tools. For a ready-made structured/vision probe, use [smoke_structured_output.py](scripts/smoke_structured_output.py).
|
||||
|
||||
Check the result, not only HTTP 200: completed text or a tool call, usable indices/dimensions, expected typed answers, or valid application JSON. Parsing JSON alone does not validate a schema. For streaming, consume deltas/items and require a terminal success; truncated output, refusals and error events are separate outcomes.
|
||||
|
||||
Start with one request and a small supported output budget. A timeout can leave an unknown billed outcome; inspect status before retrying. Compare latency and usage only for equivalent prompts, inputs, settings and successful results. Reasoning token fields, encrypted reasoning and route conversion are diagnostic evidence, not universal proof of a forced reasoning setting.
|
||||
|
||||
## Deliver
|
||||
|
||||
Identify endpoint, deployed model alias, relevant parameters, parsing/validation and observed limitations. Use the dated [verification record](references/verification.md) to distinguish source inspection, this run's live tests and untested features. A tiny smoke test proves the tested contract, not accuracy or reliability for all workloads.
|
||||
@@ -0,0 +1,4 @@
|
||||
interface:
|
||||
display_name: "EAPIL LLM API"
|
||||
short_description: "EAPIL text, retrieval and Decisions API guide"
|
||||
default_prompt: "Use $eapil-llmapi-structured to build, test or debug an EAPIL LLM API request using the appropriate endpoint."
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
# Chat, Responses, tools and streaming
|
||||
|
||||
Use the model and protocol supported by the selected channel. Examples use `gpt-6-luna`, tested on this deployment on 2026-09-30; it is an example, not a permanent default. See [verification](verification.md) for limitations.
|
||||
|
||||
## Ordinary text
|
||||
|
||||
Chat request to `/v1/chat/completions`:
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "gpt-6-luna",
|
||||
"messages": [{"role": "user", "content": "Reply with exactly PONG."}],
|
||||
"reasoning_effort": "low",
|
||||
"max_completion_tokens": 256
|
||||
}
|
||||
```
|
||||
|
||||
Read `choices[*].message.content` for text, or `message.tool_calls` for requested tools. Check `finish_reason`; `length`, content filtering and refusals are not ordinary successful answers. A `stop` with empty content and no tool call does not prove the task succeeded.
|
||||
|
||||
Responses request to `/v1/responses`:
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "gpt-6-luna",
|
||||
"input": "Reply with exactly PONG.",
|
||||
"reasoning": {"effort": "low"},
|
||||
"max_output_tokens": 256,
|
||||
"store": false
|
||||
}
|
||||
```
|
||||
|
||||
Read message items with `content[*].type == "output_text"` from `output`; an SDK's `output_text` convenience property need not exist in raw JSON. Other output items can be reasoning or function calls. Check `status`, refusal parts, `error` and `incomplete_details`. Do not parse a serialized copy of the entire response as if it were the assistant's text.
|
||||
|
||||
Reasoning effort values, token limits, sampling controls and context limits depend on the model/channel. Do not send `temperature: 0` or `reasoning: none` universally. Legacy Chat models may use `max_tokens`; current reasoning models commonly use `max_completion_tokens`. Responses uses `max_output_tokens`. These budgets may include reasoning, so a small budget can truncate before visible output.
|
||||
|
||||
## Vision
|
||||
|
||||
Chat uses:
|
||||
|
||||
```json
|
||||
{"type":"image_url","image_url":{"url":"data:image/png;base64,...","detail":"low"}}
|
||||
```
|
||||
|
||||
Responses uses:
|
||||
|
||||
```json
|
||||
{"type":"input_image","image_url":"data:image/png;base64,...","detail":"low"}
|
||||
```
|
||||
|
||||
Place these alongside `text` or `input_text` content respectively. Preserve reference order and ensure MIME matches the file. Choose image detail according to the task and model; low detail is not appropriate for every OCR or small-object task. Remote URLs, files, audio and video formats have model-specific support; a text probe does not establish them.
|
||||
|
||||
## Client function tools
|
||||
|
||||
Chat tool definition:
|
||||
|
||||
```json
|
||||
{"type":"function","function":{"name":"echo","description":"Echo a string.","parameters":{"type":"object","properties":{"text":{"type":"string"}},"required":["text"],"additionalProperties":false},"strict":true}}
|
||||
```
|
||||
|
||||
Responses tool definition:
|
||||
|
||||
```json
|
||||
{"type":"function","name":"echo","description":"Echo a string.","parameters":{"type":"object","properties":{"text":{"type":"string"}},"required":["text"],"additionalProperties":false},"strict":true}
|
||||
```
|
||||
|
||||
Chat tool choice wraps the function name in `function`; Responses puts `name` directly on the choice. This deployment accepted both shapes in live probes. Validate tool names and JSON arguments before invoking the application's authorized action. The gateway and smoke script do not execute client tools for you.
|
||||
|
||||
For continuation, Chat appends the assistant tool-call message and a `role: tool` message using the matching `tool_call_id`. Responses supplies the function-call item and `type: function_call_output`, with its matching `call_id` and string `output`. Keep relevant earlier input/output in the next request when stored state is unavailable. A `previous_response_id` depends on provider storage and routing support; `store:false` is not a guarantee of remote conversation persistence. Do not synthesize a mismatched call ID.
|
||||
|
||||
Hosted tools, WebSocket `/v1/responses`, realtime, `/v1/responses/compact`, `/v1/alpha/search` and `/v1/memories/trace_summarize` exist in the inspected source or upstream protocols, but this guide's live coverage does not establish all of them. Use their actual deployment/provider protocol before adding them to an application; do not infer support from an ordinary HTTP Responses call.
|
||||
|
||||
## SSE
|
||||
|
||||
Set `stream:true` for supported Chat/Responses requests. Chat can request `stream_options:{"include_usage":true}` when supported. Consume SSE frames, not individual network chunks; join multiline `data:` and decode complete UTF-8 lines.
|
||||
|
||||
Chat delivers content/tool argument deltas and a finishing choice, often followed by a separate usage chunk and `[DONE]`. Responses delivers typed events such as `response.output_text.delta`, output-item events and `response.completed`. Accumulate the items/deltas: on this gateway the final completed snapshot may have an empty `output` even though earlier events contain the full answer. Do not double-count `delta`, text-done and item-done copies.
|
||||
|
||||
Require a terminal success and handle refusal, failed/incomplete/error events and premature EOF separately. Do not use a received first token as proof of completion. First-token latency and total elapsed time are different metrics; the bundled probe reports total time.
|
||||
|
||||
## Smoke commands
|
||||
|
||||
Write the selected request as UTF-8 JSON in `request.json`, then run from the project. Resolve `<skill-dir>` from the actual installed location:
|
||||
|
||||
```powershell
|
||||
python '<skill-dir>/scripts/smoke_api.py' --mode chat --payload-file request.json --dry-run
|
||||
python '<skill-dir>/scripts/smoke_api.py' --mode responses --payload-file request.json --expect-text PONG
|
||||
```
|
||||
|
||||
Use the matching mode for the payload. Add `rtk proxy` in this repository. Credentials default to `EAPIL_API_KEY`, then `OPENAI_API_KEY`; `--api-key-env` selects another explicitly configured variable. Live calls make one request with no retry. Dry run prints the supplied payload, so use synthetic input when sharing that output.
|
||||
|
||||
## Sources
|
||||
|
||||
Reviewed 2026-09-30: local New-API `router/relay-router.go`, `service/relayconvert/chat_to_responses.go`, `relay/channel/openai/relay_responses.go` and mode-specific runtime probes. The source checkout includes uncommitted changes and is not a deployment identity.
|
||||
|
||||
- [OpenAI Responses API](https://developers.openai.com/api/reference/resources/responses/methods/create)
|
||||
- [OpenAI Chat Completions API](https://developers.openai.com/api/reference/resources/chat/subresources/completions/methods/create)
|
||||
- [OpenAI function calling](https://developers.openai.com/api/docs/guides/function-calling)
|
||||
- [OpenAI streaming](https://developers.openai.com/api/docs/guides/streaming-responses)
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
# Typed Decisions
|
||||
|
||||
Use Decisions for probability judgments, categorical decisions and ordinal scores. It has a separate request/response contract from Chat and Responses; it does not return an assistant message or execute the chosen business action.
|
||||
|
||||
## Gateway and model selection
|
||||
|
||||
POST `/v1/alpha/decisions` or `/api/alpha/decisions` with Bearer authentication. Use `model`, a non-null `state`, and a nonempty `questions` object. Omit `stream` or set it to `false`; this gateway rejects `stream:true`. The inspected New-API implementation routes to an OpenRouter channel's `/alpha/decisions`, preserving the request body except root model mapping. It does not automatically translate Chat/Responses requests into Decisions.
|
||||
|
||||
| Model alias | Live coverage on 2026-09-30 | Input boundary |
|
||||
|---|---|---|
|
||||
| `typesafe/jev-1.13` | `noul`, `choice`, `score` together | Ticket-shaped object tested |
|
||||
| `upstage/solar-decide` | `noul`, `choice`, `score` together | Same object tested; score can be fractional |
|
||||
| `respan/span-01` | `noul` | Conversation object below tested |
|
||||
| `respan/span-01-lite` | `noul`; `choice` rejected | Conversation object below tested |
|
||||
|
||||
The public protocol permits broader state shapes than an individual model may accept. A successful Jev request is not a portable Respan template. Respan's observed validation restricts questions to `noul`, and accepts a state string or an object containing only `input`/`output` messages. Only the conversation object was live tested here. See [verification](verification.md) for exact coverage.
|
||||
|
||||
## Classify and score in one request
|
||||
|
||||
This synthetic request passed with Jev and, replacing `model`, Solar:
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "typesafe/jev-1.13",
|
||||
"state": {"ticket": "Checkout fails after payment. Customers cannot complete orders."},
|
||||
"questions": {
|
||||
"is_bug": {
|
||||
"type": "noul",
|
||||
"instructions": "Does the ticket describe a software defect?",
|
||||
"criteria": {"true": "Broken or unexpected behavior", "false": "A question without a defect"}
|
||||
},
|
||||
"team": {
|
||||
"type": "choice",
|
||||
"instructions": "Which team should own this ticket?",
|
||||
"criteria": {"payments": "Payment and checkout issues", "account": "Login and profile issues"}
|
||||
},
|
||||
"urgency": {
|
||||
"type": "score",
|
||||
"instructions": "How urgent is the issue?",
|
||||
"criteria": ["Can wait", "Fix this week", "Blocking payment now"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Interpret `answers` by the original question keys and returned `type`:
|
||||
|
||||
- `noul`: `noul` is the probability in `[0,1]` that the condition holds, not a Boolean. Choose a task-specific threshold after evaluating relevant data.
|
||||
- `choice`: `choice` identifies a criterion key; `probabilities` maps those keys to probabilities. Preserve string labels. `confidence` is a separate field, not necessarily the largest probability.
|
||||
- `score`: `criteria` is an ordered list; zero-based positions define the scale. `probabilities` uses string index keys, and `legend`, when present, identifies the descriptions. `score` may be fractional; do not cast it to an integer merely because the criteria have integer positions.
|
||||
|
||||
Validate matching question/answer keys, types, finite numbers, ranges, probability labels and sums. The bundled probe checks the object/list criteria shapes above; additional protocol forms require corresponding handling. Preserve `usage` fields when available; monetary `cost` and token counts are different measures. One arithmetic or ticket test does not prove calibration or classification quality.
|
||||
|
||||
## Respan conversation evaluation
|
||||
|
||||
Tested with both Respan aliases:
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "respan/span-01-lite",
|
||||
"state": {
|
||||
"input": [{"role": "user", "content": "What is 2 plus 2?"}],
|
||||
"output": {"role": "assistant", "content": "The answer is 4."}
|
||||
},
|
||||
"questions": {
|
||||
"correct": {
|
||||
"type": "noul",
|
||||
"instructions": "The assistant gives the correct arithmetic answer.",
|
||||
"criteria": {"true": "Correct answer", "false": "Incorrect answer"}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The observed state validator requires string `content`, roles `system`, `user`, `assistant` or `tool`, and an assistant role for `output`. `state:{"messages":[...]}` failed. Do not put Chat's multimodal message parts or arbitrary extra state fields into this Respan object.
|
||||
|
||||
A `choice` request returned a generic error mentioning plain-string instructions/criteria. The `noul` request above with a true/false criteria object nevertheless succeeded, so that error does not establish that all criteria objects are rejected. Keep the tested shape or verify the alternative explicitly; do not extrapolate a `choice` failure into a different schema requirement.
|
||||
|
||||
## Probe
|
||||
|
||||
Save the selected synthetic request as UTF-8 `request.json`:
|
||||
|
||||
```powershell
|
||||
python '<skill-dir>/scripts/smoke_api.py' --mode decisions --payload-file request.json --dry-run
|
||||
python '<skill-dir>/scripts/smoke_api.py' --mode decisions --payload-file request.json
|
||||
python '<skill-dir>/scripts/smoke_api.py' --mode decisions --payload-file request.json --decisions-alias
|
||||
```
|
||||
|
||||
In this repository prefix external commands with `rtk proxy`. The summary includes decision labels/answers, so use synthetic questions for shared diagnostics. The helper performs no chosen action and makes no automatic retry. HTTP 400 should lead to checking model-specific state/question support, not silently substituting a Chat call.
|
||||
|
||||
## Sources
|
||||
|
||||
Reviewed 2026-09-30: [OpenRouter Decisions protocol](https://openrouter.ai/docs/api/api-reference/alphadecisions/submit-a-decisions-request), model pages for [Jev](https://openrouter.ai/typesafe/jev-1.13), [Solar](https://openrouter.ai/upstage/solar-decide) and [Respan lite](https://openrouter.ai/respan/span-01-lite). Local New-API evidence: `router/relay-router.go`, `dto/openrouter_decisions.go`, `relay/helper/valid_request.go`, `relay/openrouter_decisions_handler.go`. Deployment observations are recorded separately in [verification](verification.md).
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
# Embedding and Rerank
|
||||
|
||||
These are retrieval operations, not conversations. Use a model enabled for the actual endpoint. Model names and scores do not establish dimensionality, compatibility or calibrated probabilities.
|
||||
|
||||
## Embedding
|
||||
|
||||
POST `/v1/embeddings`:
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "text-embedding-3-small",
|
||||
"input": ["blue", "red"],
|
||||
"encoding_format": "float"
|
||||
}
|
||||
```
|
||||
|
||||
Read each `data[*].embedding` and associate it with the input using its `index`. Confirm item count, finite values, actual vector length and consistent dimensions. Preserve the embedding model/version and preprocessing used to build the index; do not mix vectors from incompatible spaces. Base64 encodings and token-array inputs need their own provider-compatible handling; the bundled probe checks float vectors from text input.
|
||||
|
||||
The live gateway returned **1024 dimensions** for this alias. Adding `dimensions:128` still returned 1024, so the dimension option was not honored in this tested route. Do not size a vector database solely from a public model name or assume every accepted parameter affects the result. Treat a required dimension mismatch as a failed contract. Other deployments/channels may behave differently.
|
||||
|
||||
`dimensions` is an optional upstream feature of certain models. This gateway's source forwards it, but code forwarding alone cannot prove implementation. Batch input order must be recovered by indices even if response rows arrive in a different order. Context/input and batch limits also depend on the provider.
|
||||
|
||||
## Rerank
|
||||
|
||||
POST `/v1/rerank`:
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "bge-reranker-v2-m3-Q8_0.gguf",
|
||||
"query": "How do I reset my password?",
|
||||
"documents": [
|
||||
"Fresh apples for sale.",
|
||||
"Open account settings and choose reset password.",
|
||||
"Rain is expected tomorrow."
|
||||
],
|
||||
"top_n": 2,
|
||||
"return_documents": true
|
||||
}
|
||||
```
|
||||
|
||||
Read `results[*].index` and `relevance_score`; the index identifies the original candidate, not its new rank. Preserve candidate IDs separately and map them by index. This deployment returned `[1,0]` for the example and scores about `1.77` and `-11.02`; these are ranking scores, not probabilities bounded by 0–1. Do not invent a universal confidence threshold across models or queries.
|
||||
|
||||
Although `return_documents:true` was supplied, this route returned only indices/scores. Recover source documents from your original list. Object-shaped documents, chunk limits and additional options depend on the selected adapter; string documents are the tested baseline. The source preserves explicit `top_n` and `return_documents` through pointer fields, including zero/false values, but their accepted semantics remain provider-specific.
|
||||
|
||||
Embedding retrieval finds candidates; reranking reorders a candidate set against the current query. Choose how many candidates to retrieve/rerank for the application's quality and cost requirements rather than inheriting a fixed number from an example.
|
||||
|
||||
## Probes
|
||||
|
||||
Put the matching request into UTF-8 `request.json`:
|
||||
|
||||
```powershell
|
||||
python '<skill-dir>/scripts/smoke_api.py' --mode embedding --payload-file request.json
|
||||
python '<skill-dir>/scripts/smoke_api.py' --mode rerank --payload-file request.json
|
||||
```
|
||||
|
||||
The helper does not dump vectors or documents. It validates indices, counts and finite numeric data; it reports actual dimensions and rejects a requested dimension mismatch. A valid vector or a sorted result is not proof of application retrieval quality.
|
||||
|
||||
## Sources
|
||||
|
||||
Reviewed 2026-09-30: local New-API `dto/embedding.go`, `dto/rerank.go`, `relay/embedding_handler.go`, `relay/rerank_handler.go`, `relay/common_handler/rerank.go` and the dated live [verification](verification.md).
|
||||
|
||||
[OpenAI embedding guide](https://developers.openai.com/api/docs/guides/embeddings) describes upstream embedding behavior; the alias-specific observations above come from this gateway's tests.
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
# Structured JSON output
|
||||
|
||||
Use for a machine-readable result, not as the default for all LLM tasks. Select a model/channel that actually supports the needed schema. The live example below passed through Responses; the same model's Chat strict-schema probe returned an empty answer. See [verification](verification.md).
|
||||
|
||||
## Distinct field shapes
|
||||
|
||||
Responses puts the format under `text.format`:
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "gpt-6-luna",
|
||||
"input": "Return ok=true and label=smoke.",
|
||||
"text": {"format": {
|
||||
"type": "json_schema", "name": "gateway_smoke", "strict": true,
|
||||
"schema": {
|
||||
"type": "object", "additionalProperties": false,
|
||||
"properties": {"ok": {"type": "boolean"}, "label": {"type": "string", "enum": ["smoke"]}},
|
||||
"required": ["ok", "label"]
|
||||
}
|
||||
}},
|
||||
"reasoning": {"effort": "low"},
|
||||
"max_output_tokens": 256,
|
||||
"store": false
|
||||
}
|
||||
```
|
||||
|
||||
The analogous Chat field is:
|
||||
|
||||
```json
|
||||
{"response_format":{"type":"json_schema","json_schema":{"name":"gateway_smoke","strict":true,"schema":{"type":"object","additionalProperties":false,"properties":{"ok":{"type":"boolean"},"label":{"type":"string","enum":["smoke"]}},"required":["ok","label"]}}}}
|
||||
```
|
||||
|
||||
Combine it with a normal Chat request's `model` and `messages`; do not copy Responses `text.format` there. This describes the protocol, not a claim that every model accepts it. A supported function tool uses its tool parameter schema instead of the final-answer format.
|
||||
|
||||
## Schema and result checks
|
||||
|
||||
Use only the JSON Schema subset supported by the selected model. Strict OpenAI examples require object fields to be listed as required and disallow additional properties; optional concepts can be represented by a nullable type where supported. Do not universally add `maxLength`, numeric bounds or array limits from an old task: supported constraints vary, and application validation still matters.
|
||||
|
||||
JSON mode (`json_object`) aims for valid JSON but does not enforce a particular schema. A prompt saying “only JSON” alone establishes neither valid syntax nor adherence. If strict enforcement is required, do not silently downgrade to prose or JSON mode after an error.
|
||||
|
||||
Extract actual output text, check completion/refusal, parse JSON, validate against the application schema with a real schema validator, and check task-specific values. Parsing any dictionary is insufficient: missing/extra keys, wrong field types, invalid enum values and truncated output must fail. Do not repair invalid business data by inventing values. JSON numbers and booleans are distinct even when a programming language treats `True == 1`.
|
||||
|
||||
The generic probe's `--expect-json` checks parsing only; it is not a general JSON Schema validator. The convenience probe validates its own small synthetic task's exact keys/types/values, not arbitrary schemas:
|
||||
|
||||
```powershell
|
||||
python '<skill-dir>/scripts/smoke_structured_output.py' --model gpt-6-luna --endpoint responses --reasoning-effort low
|
||||
```
|
||||
|
||||
`--image` can repeat for a vision-shaped smoke request, but its synthetic answer does not measure image understanding. For a visual task, verify predictions against known image content. `--dry-run` needs no key. Model selection is now explicit and reasoning defaults to omission; old CLI flags remain, but the old fixed `gpt-5.5`/`none` defaults have been removed.
|
||||
|
||||
## Diagnostics
|
||||
|
||||
Record endpoint/model/settings, success category, visible text length, elapsed time and available usage fields. A small visible answer can coexist with substantial reasoning tokens. Inspect the model's effort support, the gateway's channel overrides or conversion, and the actual usage before attributing cost to the prompt. Missing usage does not equal zero tokens; encrypted reasoning is not itself an error or proof of forced XHIGH. Keep benchmark inputs/settings equivalent and exclude failed outputs from success comparisons.
|
||||
|
||||
For HTTP 400, check the endpoint field shape, supported schema subset and model-specific options. For HTTP 200 with an empty/incomplete answer, do not mark the test as passed. Choose another verified model or endpoint deliberately when the application permits it.
|
||||
|
||||
## Sources
|
||||
|
||||
Reviewed 2026-09-30: [OpenAI structured outputs](https://developers.openai.com/api/docs/guides/structured-outputs), local conversion source and live probes described in [verification](verification.md).
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
# Verification record
|
||||
|
||||
Date: **2026-09-30**. Live target: `https://api-n-cd.playones.com`. Requests used existing environment credentials and tiny synthetic inputs. No production inputs or credentials are stored here. These observations describe the tested alias/channel/request combination; they are not promises for every deployment or model.
|
||||
|
||||
## Live requests
|
||||
|
||||
| Case | Model / endpoint | Observed result |
|
||||
|---|---|---|
|
||||
| Model discovery | GET `/v1/models` | Passed; 121 IDs returned. Listing does not establish feature support. |
|
||||
| Ordinary text | `gpt-6-luna`, Chat and Responses | Both returned exact `PONG`; normal completion. Low reasoning, 256-token budget; Responses `store:false`. |
|
||||
| Text SSE | Same model, both endpoints | Both passed exact `PONG` and terminal completion. Responses final snapshot had empty `output`; earlier deltas/items contained the answer. Parser repaired and retested successfully. |
|
||||
| Strict JSON schema | Same model, Chat | Two probes returned HTTP 200, `stop`, empty content and zero completion usage. **Failed usable-output check**; cause not established. |
|
||||
| Strict JSON schema | Same model, Responses | Passed expected keys/types/values, including the convenience script's `ok=true`, `label="smoke"`, `score=1`. |
|
||||
| Client function tools | Same model, both endpoints, nonstream | Each returned one `echo` call with parsed arguments `{"text":"ping"}`. Tool execution and continuation were not tested. |
|
||||
| Vision + strict JSON | Same model, Responses | Supplied synthetic 32×32 red PNG; returned exactly `{"color":"red"}` against red/blue enum. Chat vision not tested. |
|
||||
| Float embeddings | `text-embedding-3-small`, `/v1/embeddings` | Two text inputs returned indices 0/1 and two finite 1024-dimension vectors. |
|
||||
| Requested embedding dimension | Same alias, `dimensions:128` | HTTP 200, still 1024 dimensions. **Requested dimension not honored**; helper rejects this mismatch. |
|
||||
| Rerank | `bge-reranker-v2-m3-Q8_0.gguf`, `/v1/rerank` | Password query over three documents, `top_n:2`: indices `[1,0]`; ranking scores approximately `1.77`, `-11.02`. No documents returned despite `return_documents:true`. |
|
||||
| Three Decisions types | `typesafe/jev-1.13`, `/v1/alpha/decisions` | `noul`, `choice`, `score` together passed. Bug probability .97, team payments, urgency score 2. |
|
||||
| Three Decisions types | `upstage/solar-decide`, same endpoint | All three passed. Bug probability .984022; team payments probability .993307, confidence .942033; urgency score **1.967132**. Confidence differs from maximum probability. |
|
||||
| Respan `noul` | `respan/span-01`, `/v1/alpha/decisions`; lite, `/api/alpha/decisions` | Both passed using conversation `input`/assistant `output`; arithmetic correctness probability .95715207. Both gateway paths were exercised, not every model on both paths. |
|
||||
| Respan invalid state | lite, `state.messages` | HTTP 400; requires string state or the constrained `input`/`output` shape. |
|
||||
| Respan categorical question | lite, `choice` | HTTP 400; error states Respan accepts only `noul`. Successful `noul` used true/false criteria object. |
|
||||
| Decisions streaming | `stream:true` | HTTP 400: streaming not supported. |
|
||||
|
||||
Examples matching the inputs above are in [Chat/Responses](chat-responses.md), [structured output](structured-output.md), [retrieval](retrieval.md) and [Decisions](decisions.md). No broad model benchmark, quality evaluation, latency comparison or automatic retry was performed.
|
||||
|
||||
## Source inspection
|
||||
|
||||
Read-only source: WSL Debian `/home/code/new-api`, HEAD `8651bc59`, with existing uncommitted Decisions changes. This checkout is **not a verified identity of deployed service code**. Source was not modified.
|
||||
|
||||
- Router exposes Chat, Responses, Embedding, Rerank, `/v1/alpha/decisions` and its `/api` alias.
|
||||
- Decisions validation requires model, non-null state, nonempty questions and nonstream mode. The handler uses an OpenRouter channel, forwards the raw body with root model mapping, and checks answers/usage/cost. No Chat/Responses-to-Decisions conversion exists in the inspected code.
|
||||
- General Chat/Responses conversion exists, but channel settings and model support determine which features survive it. Source presence does not prove a deployed route works.
|
||||
- Embedding dimensions/encoding and optional Rerank values are represented and forwarded in source. Live probes demonstrate that accepted options may still be ignored downstream.
|
||||
|
||||
Responses compact/WebSocket, realtime, hosted tools, search/memory routes, stored continuation, streamed function tools, base64/token-array embeddings and other aliases were **not live verified**. Image and audio workflows remain covered by separate skills.
|
||||
|
||||
## Local regression checks
|
||||
|
||||
Run the network-mocked tests from this repository:
|
||||
|
||||
```powershell
|
||||
rtk proxy python -m unittest discover -s plugins/eapil-llmapi-structured/skills/eapil-llmapi-structured/tests -v
|
||||
```
|
||||
|
||||
All **15 tests passed**. Coverage includes wire shapes and explicit zero/false/Unicode values, endpoint joining, empty/truncated/refused outputs, UTF-8 SSE and terminal detection, tool-call identity/argument accumulation, empty final Responses snapshots, vector indices/dimensions, Rerank logits, fractional Decisions scores and concrete synthetic JSON validation. These tests validate helper behavior, independently of live service availability.
|
||||
|
||||
Skill frontmatter validation, root AGENTS.md mechanical validation, all five plugin manifests/version/assets and 27 local documentation links also passed. Plugin/skill IDs remain `eapil-llmapi-structured`; the display name is now a general LLM API guide, and the repository baseline remains 1.0.0.
|
||||
|
||||
`--expect-json` only checks JSON parsing. The convenience script checks its own fixed schema and expected values. Neither replaces application-specific schema validation or quality evaluation.
|
||||
@@ -0,0 +1,365 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Small, explicit EAPIL API probes. No retries or automatic tool execution."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib import error, parse, request
|
||||
|
||||
|
||||
PATHS = {
|
||||
"models": "models", "chat": "chat/completions", "responses": "responses",
|
||||
"embedding": "embeddings", "rerank": "rerank", "decisions": "alpha/decisions",
|
||||
}
|
||||
|
||||
|
||||
def endpoint_url(base_url: str, mode: str, decisions_alias: bool = False) -> str:
|
||||
parts = parse.urlsplit(base_url)
|
||||
if parts.scheme not in {"http", "https"} or not parts.netloc:
|
||||
raise ValueError("base URL must be HTTP or HTTPS")
|
||||
if parts.username or parts.password or parts.query or parts.fragment:
|
||||
raise ValueError("base URL must not contain credentials, query or fragment")
|
||||
base = base_url.rstrip("/")
|
||||
if decisions_alias:
|
||||
if mode != "decisions":
|
||||
raise ValueError("the /api alias applies only to decisions")
|
||||
if base.endswith(("/v1", "/api")):
|
||||
base = base.rsplit("/", 1)[0]
|
||||
return base + "/api/alpha/decisions"
|
||||
if base.endswith("/api"):
|
||||
if mode != "decisions":
|
||||
raise ValueError("use the service root or /v1 for this mode")
|
||||
elif not base.endswith("/v1"):
|
||||
base += "/v1"
|
||||
return base + "/" + PATHS[mode]
|
||||
|
||||
|
||||
def read_payload(path: str) -> dict[str, Any]:
|
||||
payload = json.loads(Path(path).read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("payload must be a JSON object")
|
||||
return payload
|
||||
|
||||
|
||||
def extract_text(mode: str, response: dict[str, Any]) -> str:
|
||||
if mode == "chat":
|
||||
choices = response.get("choices") or []
|
||||
if choices and isinstance(choices[0], dict):
|
||||
message = choices[0].get("message") or {}
|
||||
text = message.get("content") if isinstance(message, dict) else None
|
||||
return text if isinstance(text, str) else ""
|
||||
return ""
|
||||
texts = []
|
||||
for item in response.get("output") or []:
|
||||
if isinstance(item, dict) and item.get("type") == "message":
|
||||
for part in item.get("content") or []:
|
||||
if isinstance(part, dict) and part.get("type") == "output_text":
|
||||
if isinstance(part.get("text"), str):
|
||||
texts.append(part["text"])
|
||||
return "\n".join(texts)
|
||||
|
||||
|
||||
def usage_summary(response: dict[str, Any]) -> dict[str, Any]:
|
||||
usage = response.get("usage") or {}
|
||||
if not isinstance(usage, dict):
|
||||
return {}
|
||||
keys = ("input_tokens", "output_tokens", "prompt_tokens", "completion_tokens",
|
||||
"total_tokens", "cost")
|
||||
result = {key: usage[key] for key in keys if key in usage}
|
||||
for key in ("input_tokens_details", "output_tokens_details",
|
||||
"prompt_tokens_details", "completion_tokens_details"):
|
||||
if isinstance(usage.get(key), dict):
|
||||
result[key] = usage[key]
|
||||
return result
|
||||
|
||||
|
||||
def finite_number(value: Any) -> bool:
|
||||
return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value)
|
||||
|
||||
|
||||
def validate_response(mode: str, body: dict[str, Any], payload: dict[str, Any]) -> dict[str, Any]:
|
||||
if not isinstance(body, dict) or body.get("error"):
|
||||
raise ValueError("response is not a successful JSON object")
|
||||
result: dict[str, Any] = {"usage": usage_summary(body)}
|
||||
if mode == "models":
|
||||
rows = body.get("data")
|
||||
if not isinstance(rows, list):
|
||||
raise ValueError("models response has no data array")
|
||||
result["models"] = [row["id"] for row in rows if isinstance(row, dict) and isinstance(row.get("id"), str)]
|
||||
elif mode == "chat":
|
||||
choices = body.get("choices")
|
||||
if not isinstance(choices, list) or not choices:
|
||||
raise ValueError("chat response has no choices")
|
||||
reasons = [c.get("finish_reason") for c in choices if isinstance(c, dict)]
|
||||
if len(reasons) != len(choices) or any(r not in {"stop", "tool_calls", "function_call"} for r in reasons):
|
||||
raise ValueError("chat response did not finish normally")
|
||||
messages = [c.get("message") for c in choices]
|
||||
if any(not isinstance(m, dict) for m in messages):
|
||||
raise ValueError("chat response has no assistant message")
|
||||
result.update(finish_reasons=reasons, text_chars=len(extract_text(mode, body)),
|
||||
tool_calls=sum(len(m.get("tool_calls") or []) for m in messages),
|
||||
refusal=any(bool(m.get("refusal")) for m in messages))
|
||||
if result["refusal"]:
|
||||
raise ValueError("chat response refused the request")
|
||||
if not result["text_chars"] and not result["tool_calls"] and not any(m.get("function_call") for m in messages):
|
||||
raise ValueError("chat finished with no text or tool call")
|
||||
elif mode == "responses":
|
||||
if body.get("status") != "completed" or not isinstance(body.get("output"), list) or not body["output"]:
|
||||
raise ValueError("Responses result is not completed")
|
||||
result.update(status=body["status"], text_chars=len(extract_text(mode, body)),
|
||||
output_types=[i.get("type") for i in body["output"] if isinstance(i, dict)])
|
||||
if any(isinstance(c, dict) and c.get("type") == "refusal"
|
||||
for i in body["output"] if isinstance(i, dict) for c in i.get("content") or []):
|
||||
raise ValueError("Responses result refused the request")
|
||||
calls = [i for i in body["output"] if isinstance(i, dict)
|
||||
and isinstance(i.get("type"), str) and i["type"].endswith("_call")]
|
||||
if not result["text_chars"] and not calls:
|
||||
raise ValueError("Responses completed with no text or tool call")
|
||||
if any(i.get("status") in {"failed", "incomplete", "in_progress"} for i in calls):
|
||||
raise ValueError("Responses tool call did not complete")
|
||||
elif mode == "embedding":
|
||||
rows = body.get("data")
|
||||
inputs = payload.get("input")
|
||||
count = len(inputs) if isinstance(inputs, list) else 1
|
||||
if not isinstance(rows, list) or len(rows) != count or not rows:
|
||||
raise ValueError("embedding item count does not match input")
|
||||
indices = [r.get("index") for r in rows if isinstance(r, dict)]
|
||||
if len(indices) != count or any(type(i) is not int for i in indices) or sorted(indices) != list(range(count)):
|
||||
raise ValueError("embedding indices do not match input")
|
||||
vectors = [r.get("embedding") for r in rows]
|
||||
if not all(isinstance(v, list) and v and all(finite_number(x) for x in v) for v in vectors):
|
||||
raise ValueError("probe expects finite float embedding vectors; use encoding_format=float")
|
||||
dimensions = [len(v) for v in vectors]
|
||||
if len(set(dimensions)) != 1:
|
||||
raise ValueError("embedding dimensions are inconsistent")
|
||||
result.update(items=count, dimensions=dimensions, indices=indices)
|
||||
if payload.get("dimensions") is not None:
|
||||
result["requested_dimensions_honored"] = all(d == payload["dimensions"] for d in dimensions)
|
||||
if not result["requested_dimensions_honored"]:
|
||||
raise ValueError("requested embedding dimensions were not honored")
|
||||
elif mode == "rerank":
|
||||
rows = body.get("results")
|
||||
documents = payload.get("documents") or []
|
||||
if not isinstance(rows, list) or not rows:
|
||||
raise ValueError("rerank response has no results")
|
||||
indices = [r.get("index") for r in rows if isinstance(r, dict)]
|
||||
if len(indices) != len(rows) or any(type(i) is not int or not 0 <= i < len(documents) for i in indices) or len(set(indices)) != len(indices):
|
||||
raise ValueError("rerank indices do not identify the input documents")
|
||||
scores = [r.get("relevance_score") for r in rows]
|
||||
if not all(finite_number(s) for s in scores):
|
||||
raise ValueError("rerank scores are not finite numbers")
|
||||
result.update(indices=indices, scores=scores,
|
||||
documents_returned=all("document" in r for r in rows))
|
||||
elif mode == "decisions":
|
||||
questions, answers = payload.get("questions"), body.get("answers")
|
||||
if not isinstance(questions, dict) or not isinstance(answers, dict) or set(answers) != set(questions):
|
||||
raise ValueError("decision answer keys do not match question keys")
|
||||
for name, question in questions.items():
|
||||
answer = answers[name]
|
||||
if not isinstance(question, dict) or not isinstance(answer, dict) or answer.get("type") != question.get("type"):
|
||||
raise ValueError("decision answer type does not match its question")
|
||||
kind, criteria = question["type"], question.get("criteria")
|
||||
if kind == "noul":
|
||||
value = answer.get("noul")
|
||||
if not finite_number(value) or not 0 <= value <= 1:
|
||||
raise ValueError("noul is not a probability")
|
||||
elif kind in {"choice", "score"}:
|
||||
labels = set(criteria) if kind == "choice" and isinstance(criteria, dict) else {str(i) for i in range(len(criteria))} if isinstance(criteria, list) else set()
|
||||
probabilities = answer.get("probabilities")
|
||||
confidence = answer.get("confidence")
|
||||
if not labels or not isinstance(probabilities, dict) or set(probabilities) != labels:
|
||||
raise ValueError("decision probability labels do not match criteria")
|
||||
if not all(finite_number(p) and 0 <= p <= 1 for p in probabilities.values()) or not math.isclose(sum(probabilities.values()), 1, abs_tol=1e-3):
|
||||
raise ValueError("decision probabilities are invalid")
|
||||
if not finite_number(confidence) or not 0 <= confidence <= 1:
|
||||
raise ValueError("decision confidence is invalid")
|
||||
if kind == "choice" and answer.get("choice") not in labels:
|
||||
raise ValueError("decision choice is not a criterion")
|
||||
if kind == "score" and (not finite_number(answer.get("score")) or not 0 <= answer["score"] <= len(criteria) - 1):
|
||||
raise ValueError("decision score is outside the ordinal range")
|
||||
else:
|
||||
raise ValueError("unknown decision type")
|
||||
result["answers"] = answers
|
||||
return result
|
||||
|
||||
|
||||
def sse_events(response):
|
||||
data = []
|
||||
for raw_line in response:
|
||||
line = raw_line.decode("utf-8").rstrip("\r\n")
|
||||
if not line:
|
||||
if data:
|
||||
yield "\n".join(data)
|
||||
data = []
|
||||
elif line.startswith("data:"):
|
||||
data.append(line[5:].lstrip(" "))
|
||||
if data:
|
||||
yield "\n".join(data)
|
||||
|
||||
|
||||
def read_stream(response, mode: str) -> dict[str, Any]:
|
||||
if mode not in {"chat", "responses"}:
|
||||
raise ValueError("stream probe supports Chat and Responses only")
|
||||
choices, usage, done = {}, None, False
|
||||
terminal = None
|
||||
items = {}
|
||||
text_parts = {}
|
||||
for data in sse_events(response):
|
||||
if data == "[DONE]":
|
||||
done = True
|
||||
continue
|
||||
event = json.loads(data)
|
||||
if not isinstance(event, dict) or event.get("error") or event.get("type") in {"error", "response.error", "response.failed", "response.incomplete"}:
|
||||
raise ValueError("stream reported an error or incomplete result")
|
||||
if mode == "responses":
|
||||
kind = event.get("type")
|
||||
index = event.get("output_index", 0)
|
||||
if kind in {"response.output_item.added", "response.output_item.done"} and isinstance(event.get("item"), dict):
|
||||
items[index] = event["item"]
|
||||
elif kind in {"response.output_text.delta", "response.output_text.done"}:
|
||||
part_key = (index, event.get("content_index", 0))
|
||||
if kind.endswith(".delta") and isinstance(event.get("delta"), str):
|
||||
text_parts[part_key] = text_parts.get(part_key, "") + event["delta"]
|
||||
elif isinstance(event.get("text"), str):
|
||||
text_parts[part_key] = event["text"]
|
||||
if event.get("type") == "response.completed":
|
||||
terminal = event.get("response")
|
||||
else:
|
||||
if event.get("usage"):
|
||||
usage = event["usage"]
|
||||
for chunk in event.get("choices") or []:
|
||||
index = chunk.get("index", 0)
|
||||
choice = choices.setdefault(index, {"index": index, "message": {"content": ""}, "finish_reason": None})
|
||||
delta = chunk.get("delta") or {}
|
||||
if isinstance(delta.get("content"), str):
|
||||
choice["message"]["content"] += delta["content"]
|
||||
if delta.get("refusal"):
|
||||
choice["message"]["refusal"] = True
|
||||
if delta.get("tool_calls"):
|
||||
calls = choice["message"].setdefault("tool_calls", {})
|
||||
for call in delta["tool_calls"]:
|
||||
target = calls.setdefault(call.get("index", 0), {"function": {"arguments": ""}})
|
||||
for field in ("id", "type"):
|
||||
if field in call:
|
||||
target[field] = call[field]
|
||||
function = call.get("function") or {}
|
||||
if "name" in function:
|
||||
target["function"]["name"] = function["name"]
|
||||
if isinstance(function.get("arguments"), str):
|
||||
target["function"]["arguments"] += function["arguments"]
|
||||
if chunk.get("finish_reason") is not None:
|
||||
choice["finish_reason"] = chunk["finish_reason"]
|
||||
if mode == "responses":
|
||||
if not isinstance(terminal, dict):
|
||||
raise ValueError("stream ended without response.completed")
|
||||
# Some gateways omit the accumulated output from the terminal snapshot.
|
||||
if not terminal.get("output"):
|
||||
for (index, content_index), text in sorted(text_parts.items()):
|
||||
item = items.setdefault(index, {"type": "message", "role": "assistant", "content": []})
|
||||
if item.get("type") == "message":
|
||||
content = item.setdefault("content", [])
|
||||
while len(content) <= content_index:
|
||||
content.append({"type": "output_text", "text": ""})
|
||||
content[content_index] = {**content[content_index], "type": "output_text", "text": text}
|
||||
terminal["output"] = [items[i] for i in sorted(items)]
|
||||
return terminal
|
||||
if not done:
|
||||
raise ValueError("chat stream ended without [DONE]")
|
||||
for choice in choices.values():
|
||||
if isinstance(choice["message"].get("tool_calls"), dict):
|
||||
choice["message"]["tool_calls"] = list(choice["message"]["tool_calls"].values())
|
||||
return {"choices": [choices[i] for i in sorted(choices)], "usage": usage}
|
||||
|
||||
|
||||
def fetch_response(url: str, payload: dict[str, Any] | None, key: str, timeout: float, mode: str) -> dict[str, Any]:
|
||||
stream = bool(payload and payload.get("stream"))
|
||||
body = None if payload is None else json.dumps(payload, ensure_ascii=False, allow_nan=False).encode("utf-8")
|
||||
req = request.Request(url, data=body, headers={"Authorization": "Bearer " + key,
|
||||
"Content-Type": "application/json", "Accept": "text/event-stream" if stream else "application/json"},
|
||||
method="GET" if payload is None else "POST")
|
||||
with request.urlopen(req, timeout=timeout) as response:
|
||||
if stream:
|
||||
if "text/event-stream" not in response.headers.get("Content-Type", ""):
|
||||
raise ValueError("requested stream did not return text/event-stream")
|
||||
return read_stream(response, mode)
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def resolve_key(env_name: str | None) -> str:
|
||||
names = [env_name] if env_name else ["EAPIL_API_KEY", "OPENAI_API_KEY"]
|
||||
for name in names:
|
||||
value = os.getenv(name, "").strip()
|
||||
if value:
|
||||
return value
|
||||
raise ValueError("missing API key environment variable")
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--mode", choices=PATHS, required=True)
|
||||
parser.add_argument("--base-url", default="https://api-n-cd.playones.com/v1")
|
||||
parser.add_argument("--payload-file")
|
||||
parser.add_argument("--api-key-env")
|
||||
parser.add_argument("--timeout-s", type=float, default=60)
|
||||
parser.add_argument("--decisions-alias", action="store_true")
|
||||
parser.add_argument("--expect-text")
|
||||
parser.add_argument("--expect-json", action="store_true")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
key = ""
|
||||
try:
|
||||
if not math.isfinite(args.timeout_s) or args.timeout_s <= 0:
|
||||
raise ValueError("timeout must be positive and finite")
|
||||
if args.mode != "models" and not args.payload_file:
|
||||
raise ValueError("this mode requires --payload-file")
|
||||
if args.mode == "models" and args.payload_file:
|
||||
raise ValueError("models is a GET request without a payload")
|
||||
if (args.expect_json or args.expect_text is not None) and args.mode not in {"chat", "responses"}:
|
||||
raise ValueError("text expectations apply only to Chat and Responses")
|
||||
payload = read_payload(args.payload_file) if args.payload_file else None
|
||||
url = endpoint_url(args.base_url, args.mode, args.decisions_alias)
|
||||
if args.dry_run:
|
||||
print(json.dumps({"url": url, "method": "GET" if payload is None else "POST", "payload": payload}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
key = resolve_key(args.api_key_env)
|
||||
started = time.monotonic()
|
||||
body = fetch_response(url, payload, key, args.timeout_s, args.mode)
|
||||
summary = validate_response(args.mode, body, payload or {})
|
||||
if args.expect_text is not None and extract_text(args.mode, body) != args.expect_text:
|
||||
raise ValueError("output text did not match the expectation")
|
||||
if args.expect_json:
|
||||
value = json.loads(extract_text(args.mode, body))
|
||||
summary["json_kind"] = type(value).__name__
|
||||
summary.update(mode=args.mode, model=payload.get("model") if payload else None,
|
||||
stream=bool(payload and payload.get("stream")), elapsed_s=round(time.monotonic() - started, 3), ok=True)
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
except error.HTTPError as exc:
|
||||
raw = exc.read().decode("utf-8", "replace")
|
||||
try:
|
||||
parsed_error = json.loads(raw).get("error", {})
|
||||
code = parsed_error.get("code") if isinstance(parsed_error, dict) else None
|
||||
except (ValueError, AttributeError):
|
||||
code = None
|
||||
print(json.dumps({"ok": False, "http_status": exc.code, "error_code": code,
|
||||
"note": "inspect the provider error locally; raw body omitted"}), file=sys.stderr)
|
||||
return 1
|
||||
except (error.URLError, TimeoutError, OSError, ValueError, TypeError, KeyError, IndexError, AttributeError, OverflowError) as exc:
|
||||
message = str(exc)
|
||||
if key:
|
||||
message = message.replace(key, "[redacted]")
|
||||
message = re.sub(r"Bearer\s+\S+", "Bearer [redacted]", message)
|
||||
print(json.dumps({"ok": False, "error_type": type(exc).__name__, "message": message[:240]}, ensure_ascii=False), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Convenience structured/vision probe; smoke_api.py covers other API modes."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import math
|
||||
import mimetypes
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from smoke_api import endpoint_url, extract_text, fetch_response, resolve_key, validate_response
|
||||
|
||||
SCHEMA = {
|
||||
"type": "object", "additionalProperties": False,
|
||||
"properties": {"ok": {"type": "boolean"}, "label": {"type": "string"}, "score": {"type": "number"}},
|
||||
"required": ["ok", "label", "score"],
|
||||
}
|
||||
|
||||
|
||||
def data_url(path: Path) -> str:
|
||||
mime = mimetypes.guess_type(path.name)[0]
|
||||
if mime not in {"image/png", "image/jpeg", "image/webp", "image/gif"}:
|
||||
raise ValueError("image must be PNG, JPEG, WebP or GIF")
|
||||
return f"data:{mime};base64,{base64.b64encode(path.read_bytes()).decode('ascii')}"
|
||||
|
||||
|
||||
def build_payload(args) -> dict:
|
||||
prompt = 'Return one object with ok=true, label="smoke", and score=1. No markdown.'
|
||||
if args.endpoint == "responses":
|
||||
content = [{"type": "input_text", "text": prompt}]
|
||||
content.extend({"type": "input_image", "image_url": data_url(Path(p)), "detail": args.image_detail} for p in args.image)
|
||||
body = {"model": args.model, "input": [{"role": "user", "content": content}],
|
||||
"text": {"format": {"type": "json_schema", "name": "gateway_smoke", "strict": True, "schema": SCHEMA}},
|
||||
"max_output_tokens": args.max_output_tokens, "store": False}
|
||||
if args.reasoning_effort not in {"omit", "auto", "default"}:
|
||||
body["reasoning"] = {"effort": args.reasoning_effort}
|
||||
else:
|
||||
content = [{"type": "text", "text": prompt}]
|
||||
content.extend({"type": "image_url", "image_url": {"url": data_url(Path(p)), "detail": args.image_detail}} for p in args.image)
|
||||
body = {"model": args.model, "messages": [{"role": "user", "content": content}],
|
||||
"response_format": {"type": "json_schema", "json_schema": {"name": "gateway_smoke", "strict": True, "schema": SCHEMA}},
|
||||
"max_completion_tokens": args.max_output_tokens}
|
||||
if args.reasoning_effort not in {"omit", "auto", "default"}:
|
||||
body["reasoning_effort"] = args.reasoning_effort
|
||||
return body
|
||||
|
||||
|
||||
def validate_smoke_object(value) -> None:
|
||||
if not isinstance(value, dict) or set(value) != {"ok", "label", "score"}:
|
||||
raise ValueError("structured result has wrong keys")
|
||||
if type(value["ok"]) is not bool or not isinstance(value["label"], str):
|
||||
raise ValueError("structured result has wrong field types")
|
||||
score = value["score"]
|
||||
if isinstance(score, bool) or not isinstance(score, (int, float)) or not math.isfinite(score):
|
||||
raise ValueError("structured score must be finite")
|
||||
if value != {"ok": True, "label": "smoke", "score": 1}:
|
||||
raise ValueError("structured result did not match the synthetic task")
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="https://api-n-cd.playones.com/v1")
|
||||
parser.add_argument("--model", required=True)
|
||||
parser.add_argument("--endpoint", choices=["responses", "chat"], default="responses")
|
||||
parser.add_argument("--reasoning-effort", default="omit")
|
||||
parser.add_argument("--max-output-tokens", type=int, default=256)
|
||||
parser.add_argument("--image", action="append", default=[])
|
||||
parser.add_argument("--image-detail", default="low")
|
||||
parser.add_argument("--api-key-env")
|
||||
parser.add_argument("--timeout-s", type=float, default=60)
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
key = ""
|
||||
try:
|
||||
if args.max_output_tokens <= 0 or not math.isfinite(args.timeout_s) or args.timeout_s <= 0:
|
||||
raise ValueError("token limit and timeout must be positive")
|
||||
body = build_payload(args)
|
||||
url = endpoint_url(args.base_url, args.endpoint)
|
||||
if args.dry_run:
|
||||
print(json.dumps({"url": url, "payload": body}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
key = resolve_key(args.api_key_env)
|
||||
started = time.monotonic()
|
||||
response = fetch_response(url, body, key, args.timeout_s, args.endpoint)
|
||||
summary = validate_response(args.endpoint, response, body)
|
||||
validate_smoke_object(json.loads(extract_text(args.endpoint, response)))
|
||||
summary.update(ok=True, schema_checked=True, endpoint=args.endpoint,
|
||||
model=args.model, elapsed_s=round(time.monotonic()-started, 3))
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
except Exception as exc:
|
||||
# Deliberately avoid raw provider bodies, prompts, images and credentials.
|
||||
message = str(exc).replace(key, "[redacted]") if key else str(exc)
|
||||
report = {"ok": False, "error_type": type(exc).__name__}
|
||||
if hasattr(exc, "code"):
|
||||
report["http_status"] = exc.code
|
||||
else:
|
||||
report["message"] = message[:240]
|
||||
print(json.dumps(report, ensure_ascii=False), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+195
@@ -0,0 +1,195 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
||||
import smoke_api as api
|
||||
import smoke_structured_output as structured
|
||||
|
||||
|
||||
def stream(*events):
|
||||
lines = []
|
||||
for event in events:
|
||||
value = event if isinstance(event, str) else json.dumps(event, ensure_ascii=False)
|
||||
lines.append(("data: " + value + "\r\n\r\n").encode("utf-8"))
|
||||
return io.BytesIO(b"".join(lines))
|
||||
|
||||
|
||||
class APIProbeTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.network = mock.patch.object(api.request, "urlopen", side_effect=AssertionError("unexpected network"))
|
||||
self.urlopen = self.network.start()
|
||||
self.addCleanup(self.network.stop)
|
||||
|
||||
def test_endpoint_joining_preserves_prefix_and_supports_decisions_alias(self):
|
||||
for base in ("https://example.test", "https://example.test/v1", "https://example.test/v1/"):
|
||||
self.assertEqual(api.endpoint_url(base, "chat"), "https://example.test/v1/chat/completions")
|
||||
self.assertEqual(api.endpoint_url("https://example.test/relay/v1", "responses"), "https://example.test/relay/v1/responses")
|
||||
self.assertEqual(api.endpoint_url("https://example.test/v1", "decisions", True), "https://example.test/api/alpha/decisions")
|
||||
self.assertEqual(api.endpoint_url("https://example.test/api", "decisions"), "https://example.test/api/alpha/decisions")
|
||||
for base in ("https://key@example.test", "https://example.test?key=secret", "file:///tmp"):
|
||||
with self.subTest(base=base), self.assertRaises(ValueError):
|
||||
api.endpoint_url(base, "chat")
|
||||
|
||||
def test_wire_request_preserves_zero_false_unicode_and_exact_mode_shape(self):
|
||||
payload = {"model": "alias", "messages": [{"role": "user", "content": '中文 "quoted" $HOME'}], "temperature": 0, "stream": False}
|
||||
response = io.BytesIO(b'{"choices":[]}')
|
||||
response.headers = {"Content-Type": "application/json"}
|
||||
self.urlopen.side_effect = None
|
||||
self.urlopen.return_value = response
|
||||
api.fetch_response("https://example.test/v1/chat/completions", payload, "test-key", 12, "chat")
|
||||
req = self.urlopen.call_args.args[0]
|
||||
self.assertEqual(req.get_method(), "POST")
|
||||
self.assertEqual(req.get_header("Authorization"), "Bearer test-key")
|
||||
self.assertEqual(json.loads(req.data), payload)
|
||||
self.assertEqual(self.urlopen.call_args.kwargs["timeout"], 12)
|
||||
|
||||
def test_models_is_get_and_has_no_body(self):
|
||||
response = io.BytesIO(b'{"data":[]}')
|
||||
response.headers = {}
|
||||
self.urlopen.side_effect = None
|
||||
self.urlopen.return_value = response
|
||||
api.fetch_response("https://example.test/v1/models", None, "test-key", 12, "models")
|
||||
req = self.urlopen.call_args.args[0]
|
||||
self.assertEqual(req.get_method(), "GET")
|
||||
self.assertIsNone(req.data)
|
||||
|
||||
def test_responses_text_does_not_fall_back_to_serializing_raw_response(self):
|
||||
self.assertEqual(api.extract_text("responses", {"output": [{"type": "reasoning", "content": [{"text": "private reasoning"}]}]}), "")
|
||||
self.assertEqual(api.extract_text("responses", {"output": [{"type": "message", "content": [{"type": "output_text", "text": "答案"}]}]}), "答案")
|
||||
|
||||
def test_empty_chat_truncation_refusal_and_incomplete_responses_fail(self):
|
||||
bad = [
|
||||
("chat", {"choices": [{"message": {"content": ""}, "finish_reason": "stop"}]}),
|
||||
("chat", {"choices": [{"message": {"content": "partial"}, "finish_reason": "length"}]}),
|
||||
("chat", {"choices": [{"message": {"refusal": "refused"}, "finish_reason": "stop"}]}),
|
||||
("responses", {"status": "incomplete", "output": []}),
|
||||
("responses", {"status": "completed", "output": []}),
|
||||
("responses", {"status": "completed", "output": [{"type": "reasoning"}]}),
|
||||
("responses", {"status": "completed", "output": [{"type": "message", "content": []}]}),
|
||||
("responses", {"status": "completed", "output": [{"type": "web_search_call", "status": "failed"}]}),
|
||||
("responses", {"status": "completed", "output": [{"type": "message", "content": [{"type": "refusal", "refusal": "no"}]}]}),
|
||||
]
|
||||
for mode, body in bad:
|
||||
with self.subTest(mode=mode, body=body), self.assertRaises(ValueError):
|
||||
api.validate_response(mode, body, {})
|
||||
|
||||
def test_tool_only_chat_and_responses_are_valid_without_text(self):
|
||||
chat = {"choices": [{"message": {"content": None, "tool_calls": [{"function": {"name": "echo", "arguments": "{}"}}]}, "finish_reason": "tool_calls"}]}
|
||||
self.assertEqual(api.validate_response("chat", chat, {})["tool_calls"], 1)
|
||||
response = {"status": "completed", "output": [{"type": "function_call", "name": "echo", "arguments": "{}"}]}
|
||||
self.assertEqual(api.validate_response("responses", response, {})["output_types"], ["function_call"])
|
||||
|
||||
def test_chat_sse_assembles_utf8_and_usage_and_requires_done(self):
|
||||
events = [
|
||||
{"choices": [{"index": 0, "delta": {"content": "中"}, "finish_reason": None}]},
|
||||
{"choices": [{"index": 0, "delta": {"content": "文"}, "finish_reason": "stop"}]},
|
||||
{"choices": [], "usage": {"total_tokens": 5}},
|
||||
]
|
||||
result = api.read_stream(stream(*events, "[DONE]"), "chat")
|
||||
self.assertEqual(api.extract_text("chat", result), "中文")
|
||||
self.assertEqual(api.validate_response("chat", result, {})["usage"]["total_tokens"], 5)
|
||||
with self.assertRaises(ValueError):
|
||||
api.read_stream(stream(*events), "chat")
|
||||
|
||||
def test_chat_tool_sse_preserves_call_identity_and_joins_arguments(self):
|
||||
result = api.read_stream(stream(
|
||||
{"choices": [{"index": 0, "delta": {"tool_calls": [{"index": 0, "id": "call_1", "type": "function", "function": {"name": "echo", "arguments": '{"text":'}}]}}]},
|
||||
{"choices": [{"index": 0, "delta": {"tool_calls": [{"index": 0, "function": {"arguments": '"ping"}'}}]}, "finish_reason": "tool_calls"}]},
|
||||
"[DONE]",
|
||||
), "chat")
|
||||
call = result["choices"][0]["message"]["tool_calls"][0]
|
||||
self.assertEqual(call["id"], "call_1")
|
||||
self.assertEqual(call["type"], "function")
|
||||
self.assertEqual(call["function"]["name"], "echo")
|
||||
self.assertEqual(json.loads(call["function"]["arguments"]), {"text": "ping"})
|
||||
self.assertEqual(api.validate_response("chat", result, {})["tool_calls"], 1)
|
||||
|
||||
def test_responses_sse_recovers_items_when_terminal_output_is_empty(self):
|
||||
item = {"type": "message", "content": [{"type": "output_text", "text": "PONG"}]}
|
||||
result = api.read_stream(stream(
|
||||
{"type": "response.output_text.delta", "output_index": 0, "content_index": 0, "delta": "P"},
|
||||
{"type": "response.output_text.delta", "output_index": 0, "content_index": 0, "delta": "ONG"},
|
||||
{"type": "response.output_text.done", "output_index": 0, "content_index": 0, "text": "PONG"},
|
||||
{"type": "response.output_item.done", "output_index": 0, "item": item},
|
||||
{"type": "response.completed", "response": {"status": "completed", "output": [], "usage": {"output_tokens": 6}}},
|
||||
), "responses")
|
||||
self.assertEqual(api.extract_text("responses", result), "PONG")
|
||||
self.assertEqual(api.validate_response("responses", result, {})["usage"]["output_tokens"], 6)
|
||||
|
||||
def test_responses_sse_deltas_without_item_snapshot_and_failure_terminals(self):
|
||||
result = api.read_stream(stream(
|
||||
{"type": "response.output_text.delta", "output_index": 0, "content_index": 0, "delta": "answer"},
|
||||
{"type": "response.completed", "response": {"status": "completed", "output": []}},
|
||||
), "responses")
|
||||
self.assertEqual(api.extract_text("responses", result), "answer")
|
||||
for kind in ("error", "response.failed", "response.incomplete"):
|
||||
with self.subTest(kind=kind), self.assertRaises(ValueError):
|
||||
api.read_stream(stream({"type": kind}), "responses")
|
||||
with self.assertRaises(ValueError):
|
||||
api.read_stream(stream({"type": "response.output_text.delta", "delta": "partial"}), "responses")
|
||||
|
||||
def test_embedding_indices_dimensions_and_finite_vectors(self):
|
||||
payload = {"input": ["one", "two"]}
|
||||
body = {"data": [{"index": 1, "embedding": [0.1, 0.2]}, {"index": 0, "embedding": [0.3, 0.4]}]}
|
||||
self.assertEqual(api.validate_response("embedding", body, payload)["dimensions"], [2, 2])
|
||||
with self.assertRaises(ValueError):
|
||||
api.validate_response("embedding", body, dict(payload, dimensions=128))
|
||||
for vector in ([], [float("nan")], [True], "base64"):
|
||||
with self.subTest(vector=vector), self.assertRaises(ValueError):
|
||||
api.validate_response("embedding", {"data": [{"index": 0, "embedding": vector}]}, {"input": "one"})
|
||||
with self.assertRaises(ValueError):
|
||||
api.validate_response("embedding", {"data": [{"index": False, "embedding": [1]}]}, {"input": "one"})
|
||||
|
||||
def test_rerank_accepts_logits_and_maps_original_indices(self):
|
||||
result = api.validate_response("rerank", {"results": [{"index": 1, "relevance_score": 1.77}, {"index": 0, "relevance_score": -11.02}]}, {"documents": ["apple", "password"]})
|
||||
self.assertEqual(result["indices"], [1, 0])
|
||||
self.assertFalse(result["documents_returned"])
|
||||
for index in (2, -1, True):
|
||||
with self.subTest(index=index), self.assertRaises(ValueError):
|
||||
api.validate_response("rerank", {"results": [{"index": index, "relevance_score": 1}]}, {"documents": ["one"]})
|
||||
|
||||
def test_decision_fractional_score_confidence_and_question_correspondence(self):
|
||||
payload = {"questions": {
|
||||
"team": {"type": "choice", "criteria": {"a": "one", "b": "two"}},
|
||||
"urgency": {"type": "score", "criteria": ["wait", "soon", "now"]},
|
||||
"correct": {"type": "noul", "criteria": "correct answer"},
|
||||
}}
|
||||
answers = {"team": {"type": "choice", "choice": "a", "probabilities": {"a": .99, "b": .01}, "confidence": .94},
|
||||
"urgency": {"type": "score", "score": 1.97, "probabilities": {"0": .01, "1": .01, "2": .98}, "confidence": .87},
|
||||
"correct": {"type": "noul", "noul": .96}}
|
||||
self.assertEqual(api.validate_response("decisions", {"answers": answers}, payload)["answers"], answers)
|
||||
with self.assertRaises(ValueError):
|
||||
api.validate_response("decisions", {"answers": {"correct": answers["correct"]}}, payload)
|
||||
bad = {**answers, "correct": {"type": "noul", "noul": 2}}
|
||||
with self.assertRaises(ValueError):
|
||||
api.validate_response("decisions", {"answers": bad}, payload)
|
||||
|
||||
def test_structured_validation_does_not_confuse_boolean_with_numeric_or_accept_extra_keys(self):
|
||||
structured.validate_smoke_object({"ok": True, "label": "smoke", "score": 1})
|
||||
for value in ({"ok": 1, "label": "smoke", "score": 1},
|
||||
{"ok": True, "label": "smoke", "score": True},
|
||||
{"ok": True, "label": "smoke", "score": 1, "extra": 0}):
|
||||
with self.subTest(value=value), self.assertRaises(ValueError):
|
||||
structured.validate_smoke_object(value)
|
||||
|
||||
def test_dry_run_needs_no_key_makes_no_network_call_and_reads_bom(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "payload.json"
|
||||
path.write_text(json.dumps({"model": "alias", "input": "中文"}), encoding="utf-8-sig")
|
||||
output = io.StringIO()
|
||||
with contextlib.redirect_stdout(output), mock.patch.dict(api.os.environ, {}, clear=True):
|
||||
self.assertEqual(api.main(["--mode", "responses", "--payload-file", str(path), "--dry-run"]), 0)
|
||||
self.assertEqual(json.loads(output.getvalue())["payload"]["input"], "中文")
|
||||
self.urlopen.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user