4e854ba8c2
Native /login with a verified API key, automatic model discovery from UniGPT's /models endpoint, per-deployment deployment-id headers, and model metadata from pi's bundled catalog.
372 lines
17 KiB
TypeScript
372 lines
17 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { once } from "node:events";
|
|
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
|
import { test, type TestContext } from "node:test";
|
|
import {
|
|
createModels,
|
|
InMemoryCredentialStore,
|
|
InMemoryModelsStore,
|
|
type AuthEvent,
|
|
type AuthInteraction,
|
|
} from "@earendil-works/pi-ai";
|
|
import {
|
|
BASE_URL,
|
|
createUnigptProvider,
|
|
FALLBACK_CONTEXT_WINDOW,
|
|
FALLBACK_MAX_TOKENS,
|
|
fetchCatalog,
|
|
lookupCatalogMetadata,
|
|
makeModel,
|
|
parseCatalog,
|
|
type ModelMetadata,
|
|
} from "../src/provider.ts";
|
|
|
|
const KEY = "test-key-not-a-real-secret";
|
|
const signal = () => new AbortController().signal;
|
|
const catalog = (...ids: string[]) => Response.json({ data: ids.map((id) => ({ id })) });
|
|
|
|
function interaction(key = KEY, events: AuthEvent[] = []): AuthInteraction {
|
|
return { prompt: async () => key, notify: (event) => events.push(event) };
|
|
}
|
|
|
|
const ids = async (...args: Parameters<typeof fetchCatalog>) => (await fetchCatalog(...args)).ids;
|
|
|
|
async function runtime(fetchImpl: typeof fetch, withCredential = true) {
|
|
const credentials = new InMemoryCredentialStore();
|
|
if (withCredential) {
|
|
await credentials.modify("unigpt", async () => ({ type: "api_key", key: KEY }));
|
|
}
|
|
const modelsStore = new InMemoryModelsStore();
|
|
const models = createModels({ credentials, modelsStore });
|
|
const provider = createUnigptProvider({ fetch: fetchImpl });
|
|
models.setProvider(provider);
|
|
return { models, provider, credentials, modelsStore };
|
|
}
|
|
|
|
test("parses IDs, deduplicates in catalog order, and permits an empty catalog", () => {
|
|
assert.deepEqual(parseCatalog({ data: [{ id: "second" }, { id: "first" }, { id: "second" }] }), { ids: ["second", "first"], skipped: 0 });
|
|
assert.deepEqual(parseCatalog({ data: [] }), { ids: [], skipped: 0 });
|
|
});
|
|
|
|
test("skips unusable entries but rejects malformed or entirely unusable catalogs", () => {
|
|
assert.deepEqual(
|
|
parseCatalog({ data: [{ id: "good" }, null, {}, { id: 42 }, { id: "" }, { id: " padded " }, { id: "header\ninjection" }] }),
|
|
{ ids: ["good"], skipped: 6 },
|
|
);
|
|
for (const payload of [null, [], {}, { data: {} }, { data: [null] }, { data: [{ id: "a\r\nb" }] }]) {
|
|
assert.throws(() => parseCatalog(payload), /invalid|without any valid/);
|
|
}
|
|
});
|
|
|
|
const known: Record<string, ModelMetadata> = {
|
|
"model-a": { name: "Model A", input: ["text", "image"], contextWindow: 272_000, maxTokens: 128_000 },
|
|
};
|
|
const lookup = (id: string) => known[id];
|
|
|
|
test("known deployments take pi's catalog metadata; each gets its own header", () => {
|
|
const a = makeModel("model-a", BASE_URL, lookup);
|
|
const b = makeModel("model-b", BASE_URL, lookup);
|
|
assert.deepEqual(a.headers, { "deployment-id": "model-a" });
|
|
assert.deepEqual(b.headers, { "deployment-id": "model-b" });
|
|
assert.equal(a.provider, "unigpt");
|
|
assert.equal(a.api, "openai-responses");
|
|
assert.equal(a.name, "Model A");
|
|
assert.deepEqual(a.input, ["text", "image"]);
|
|
assert.equal(a.contextWindow, 272_000);
|
|
assert.equal(a.maxTokens, 128_000);
|
|
// UniGPT ignores reasoning.effort, so no deployment advertises thinking controls.
|
|
assert.equal(a.reasoning, false);
|
|
assert.deepEqual(a.cost, { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 });
|
|
assert.equal(a.promptCache, undefined);
|
|
});
|
|
|
|
test("unknown deployments fall back to pi's custom-model defaults", () => {
|
|
const b = makeModel("model-b", BASE_URL, lookup);
|
|
assert.equal(b.name, "model-b");
|
|
assert.deepEqual(b.input, ["text"]);
|
|
assert.equal(b.contextWindow, FALLBACK_CONTEXT_WINDOW);
|
|
assert.equal(b.maxTokens, FALLBACK_MAX_TOKENS);
|
|
assert.equal(b.reasoning, false);
|
|
});
|
|
|
|
test("catalog lookup uses pi's bundled OpenAI/Azure entries and takes the conservative limit", () => {
|
|
const gpt4o = lookupCatalogMetadata("gpt-4o");
|
|
assert.ok(gpt4o, "pi's catalog should know gpt-4o");
|
|
assert.deepEqual(gpt4o.input, ["text", "image"]);
|
|
assert.ok(gpt4o.contextWindow >= 100_000 && gpt4o.maxTokens >= 4_096);
|
|
assert.equal(lookupCatalogMetadata("definitely-not-a-model"), undefined);
|
|
});
|
|
|
|
test("discovery uses bearer auth, no deployment header, a timeout signal, and no redirects", async () => {
|
|
const fakeFetch: typeof fetch = async (url, options) => {
|
|
assert.equal(url, BASE_URL + "/models");
|
|
const headers = new Headers(options?.headers);
|
|
assert.equal(headers.get("Authorization"), `Bearer ${KEY}`);
|
|
assert.equal(headers.get("deployment-id"), null);
|
|
assert.equal(headers.get("Accept"), "application/json");
|
|
assert.equal(options?.redirect, "error");
|
|
assert.ok(options?.signal instanceof AbortSignal);
|
|
return catalog("model-a");
|
|
};
|
|
assert.deepEqual(await ids(BASE_URL + "///", KEY, signal(), fakeFetch), ["model-a"]);
|
|
});
|
|
|
|
test("unconfigured providers do not fetch or become available", async () => {
|
|
let calls = 0;
|
|
const { models } = await runtime(async () => { calls++; return catalog("a"); }, false);
|
|
assert.deepEqual(await models.getAvailable("unigpt"), []);
|
|
assert.equal(await models.getAuth("unigpt"), undefined);
|
|
assert.equal((await models.refresh()).errors.size, 0);
|
|
assert.equal(calls, 0);
|
|
});
|
|
|
|
test("native login masks input, validates access, and lets pi persist the credential", async () => {
|
|
let calls = 0;
|
|
const events: AuthEvent[] = [];
|
|
const { models, credentials } = await runtime(async () => { calls++; return catalog("a", "b"); }, false);
|
|
await models.login("unigpt", "api_key", {
|
|
prompt: async (prompt) => {
|
|
assert.equal(prompt.type, "secret");
|
|
return ` ${KEY} `;
|
|
},
|
|
notify: (event) => events.push(event),
|
|
});
|
|
assert.equal(calls, 1);
|
|
assert.ok(!JSON.stringify(events).includes(KEY));
|
|
assert.ok(events.some((e) => e.type === "info" && e.links?.some((link) => link.url.startsWith("https://unigpt.uni-graz.at"))));
|
|
assert.ok(events.some((e) => e.type === "info" && /2 UniGPT models available/.test(e.message)));
|
|
assert.deepEqual(await credentials.read("unigpt"), { type: "api_key", key: KEY });
|
|
assert.equal((await models.refresh()).errors.size, 0);
|
|
assert.deepEqual(models.getModels("unigpt").map((m) => m.id), ["a", "b"]);
|
|
assert.equal((await models.getAvailable("unigpt")).length, 2);
|
|
await models.logout("unigpt");
|
|
assert.equal(await credentials.read("unigpt"), undefined);
|
|
assert.deepEqual(await models.getAvailable("unigpt"), []);
|
|
});
|
|
|
|
test("empty and rejected API keys never overwrite a saved credential", async () => {
|
|
let calls = 0;
|
|
const { models, credentials } = await runtime(async () => { calls++; return new Response("private upstream body", { status: 401 }); });
|
|
await assert.rejects(models.login("unigpt", "api_key", interaction(" ")), /required/);
|
|
assert.equal(calls, 0);
|
|
await assert.rejects(models.login("unigpt", "api_key", interaction("invalid-new-key")), /401/);
|
|
assert.deepEqual(await credentials.read("unigpt"), { type: "api_key", key: KEY });
|
|
});
|
|
|
|
test("login with an empty catalog still saves the key but explains the empty model list", async () => {
|
|
const events: AuthEvent[] = [];
|
|
const { models, credentials } = await runtime(async () => catalog(), false);
|
|
await models.login("unigpt", "api_key", interaction(KEY, events));
|
|
assert.deepEqual(await credentials.read("unigpt"), { type: "api_key", key: KEY });
|
|
assert.ok(events.some((e) => e.type === "info" && /lists no models/.test(e.message)));
|
|
});
|
|
|
|
test("cancelled login never saves a key or sends validation traffic", async () => {
|
|
const controller = new AbortController();
|
|
let calls = 0;
|
|
const { models, credentials } = await runtime(async () => { calls++; return catalog("a"); }, false);
|
|
await assert.rejects(models.login("unigpt", "api_key", {
|
|
signal: controller.signal,
|
|
prompt: async () => { controller.abort(); return KEY; },
|
|
notify: () => {},
|
|
}), { name: "AbortError" });
|
|
assert.equal(calls, 0);
|
|
assert.equal(await credentials.read("unigpt"), undefined);
|
|
});
|
|
|
|
test("discovery errors are useful but never echo secrets or server error bodies", async () => {
|
|
for (const status of [401, 403, 404, 429, 500]) {
|
|
await assert.rejects(fetchCatalog(BASE_URL, KEY, signal(), async () => new Response(KEY, { status })), (error: Error) => {
|
|
assert.match(error.message, new RegExp(String(status)));
|
|
assert.ok(!error.message.includes(KEY));
|
|
return true;
|
|
});
|
|
}
|
|
await assert.rejects(fetchCatalog(BASE_URL, KEY, signal(), async () => { throw new Error(KEY); }), (error: Error) => {
|
|
assert.match(error.message, /Could not reach/);
|
|
assert.ok(!error.message.includes(KEY));
|
|
return true;
|
|
});
|
|
await assert.rejects(fetchCatalog(BASE_URL, KEY, signal(), async () => new Response("not-json")), /not valid JSON/);
|
|
});
|
|
|
|
test("rate limits surface Retry-After", async () => {
|
|
const limited = async () => new Response(null, { status: 429, headers: { "Retry-After": "30" } });
|
|
await assert.rejects(fetchCatalog(BASE_URL, KEY, signal(), limited), /rate-limiting.*Retry in 30s/);
|
|
});
|
|
|
|
test("a stalled endpoint produces a readable timeout, not an AbortError", async () => {
|
|
const stalled: typeof fetch = (_url, options) => new Promise<Response>((_resolve, reject) => {
|
|
options!.signal!.addEventListener("abort", () => reject(options!.signal!.reason), { once: true });
|
|
});
|
|
await assert.rejects(fetchCatalog(BASE_URL, KEY, signal(), stalled, 20), (error: Error) => {
|
|
assert.equal(error.name, "Error");
|
|
assert.match(error.message, /did not respond within/);
|
|
return true;
|
|
});
|
|
});
|
|
|
|
test("discovery propagates cancellation to fetch", async () => {
|
|
const controller = new AbortController();
|
|
let started!: () => void;
|
|
const ready = new Promise<void>((resolve) => { started = resolve; });
|
|
const promise = fetchCatalog(BASE_URL, KEY, controller.signal, async (_url: unknown, options?: RequestInit) => {
|
|
started();
|
|
return new Promise<Response>((_resolve, reject) => {
|
|
options!.signal!.addEventListener("abort", () => reject(options!.signal!.reason), { once: true });
|
|
});
|
|
});
|
|
await ready;
|
|
controller.abort();
|
|
await assert.rejects(promise, { name: "AbortError" });
|
|
});
|
|
|
|
test("refresh persists catalog metadata only, restores offline, and keeps cache on failure", async () => {
|
|
let fail = false;
|
|
const { models, provider, credentials, modelsStore } = await runtime(async () => {
|
|
if (fail) return new Response("unavailable", { status: 503 });
|
|
return catalog("a", "b");
|
|
});
|
|
assert.equal((await models.refresh()).errors.size, 0);
|
|
const stored = await modelsStore.read("unigpt");
|
|
assert.ok(stored);
|
|
assert.ok(!JSON.stringify(stored).includes(KEY));
|
|
assert.deepEqual(stored.models.map((m) => m.id), ["a", "b"]);
|
|
fail = true;
|
|
assert.equal((await models.refresh()).errors.size, 1);
|
|
assert.deepEqual(provider.getModels().map((m) => m.id), ["a", "b"]);
|
|
|
|
let calls = 0;
|
|
const offline = createModels({ credentials, modelsStore });
|
|
offline.setProvider(createUnigptProvider({ fetch: async () => { calls++; throw new Error("no network"); } }));
|
|
assert.equal((await offline.refresh({ allowNetwork: false })).errors.size, 0);
|
|
assert.equal(calls, 0);
|
|
assert.deepEqual(offline.getModels("unigpt").map((m) => m.id), ["a", "b"]);
|
|
});
|
|
|
|
test("malformed refresh preserves the previous catalog; partial and valid refreshes replace it", async () => {
|
|
let response = () => catalog("a", "b");
|
|
const { models } = await runtime(async () => response());
|
|
await models.refresh();
|
|
response = () => Response.json({ data: [{}, null] });
|
|
assert.equal((await models.refresh()).errors.size, 1);
|
|
assert.deepEqual(models.getModels("unigpt").map((m) => m.id), ["a", "b"]);
|
|
response = () => Response.json({ data: [{ id: "a" }, {}] });
|
|
assert.equal((await models.refresh()).errors.size, 0);
|
|
assert.deepEqual(models.getModels("unigpt").map((m) => m.id), ["a"]);
|
|
response = () => catalog("b", "c");
|
|
await models.refresh();
|
|
assert.deepEqual(models.getModels("unigpt").map((m) => m.id), ["b", "c"]);
|
|
response = () => catalog();
|
|
await models.refresh();
|
|
assert.deepEqual(models.getModels("unigpt"), []);
|
|
});
|
|
|
|
test("discovery honors the effective base URL supplied by pi", async () => {
|
|
const alternate = "https://gateway.example.test/v1";
|
|
const provider = createUnigptProvider({ fetch: async (url) => {
|
|
assert.equal(url, alternate + "/models");
|
|
return catalog("a");
|
|
} });
|
|
await provider.refreshModels!({
|
|
credential: { type: "api_key", key: KEY },
|
|
baseUrl: alternate,
|
|
signal: signal(),
|
|
allowNetwork: true,
|
|
publish: async (publication) => { publication.update?.(); return true; },
|
|
});
|
|
assert.equal(provider.getModels()[0].baseUrl, alternate);
|
|
});
|
|
|
|
/** A local Responses endpoint plus a configured runtime whose catalog lists `ids`. */
|
|
async function localGateway(t: TestContext, ids: string[], handler: (req: IncomingMessage, res: ServerResponse) => void) {
|
|
const server = createServer(handler);
|
|
server.listen(0, "127.0.0.1");
|
|
await once(server, "listening");
|
|
t.after(() => new Promise<void>((resolve) => { server.closeAllConnections(); server.close(() => resolve()); }));
|
|
const address = server.address();
|
|
assert.ok(address && typeof address !== "string");
|
|
const credentials = new InMemoryCredentialStore();
|
|
await credentials.modify("unigpt", async () => ({ type: "api_key", key: KEY }));
|
|
const models = createModels({ credentials });
|
|
models.setProvider(createUnigptProvider({
|
|
baseUrl: `http://127.0.0.1:${address.port}/v1`,
|
|
fetch: async () => catalog(...ids),
|
|
lookup: () => undefined,
|
|
}));
|
|
await models.refresh();
|
|
return models;
|
|
}
|
|
|
|
function sendEvents(res: ServerResponse, events: unknown[]) {
|
|
res.writeHead(200, { "Content-Type": "text/event-stream" });
|
|
for (const event of events) res.write(`data: ${JSON.stringify(event)}\n\n`);
|
|
res.end("data: [DONE]\n\n");
|
|
}
|
|
|
|
test("real Responses transport sends deployment/auth headers, streams text, and records usage", async (t) => {
|
|
const requests: { url?: string; headers: IncomingMessage["headers"]; body: Record<string, unknown> }[] = [];
|
|
const models = await localGateway(t, ["deployment-a", "deployment-b"], (req, res) => {
|
|
let body = "";
|
|
req.on("data", (chunk) => { body += chunk; });
|
|
req.on("end", () => {
|
|
requests.push({ url: req.url, headers: req.headers, body: JSON.parse(body) });
|
|
const item = { id: "msg_1", type: "message", role: "assistant", status: "completed", content: [{ type: "output_text", text: "Hello!", annotations: [] }] };
|
|
sendEvents(res, [
|
|
{ type: "response.created", response: { id: "resp_1", status: "in_progress" } },
|
|
{ type: "response.output_item.added", output_index: 0, item: { ...item, content: [] } },
|
|
{ type: "response.output_text.delta", item_id: "msg_1", output_index: 0, content_index: 0, delta: "Hello!" },
|
|
{ type: "response.output_item.done", output_index: 0, item },
|
|
{ type: "response.completed", response: { id: "resp_1", status: "completed", output: [item], usage: { input_tokens: 12, output_tokens: 3, total_tokens: 15 } } },
|
|
]);
|
|
});
|
|
});
|
|
for (const id of ["deployment-a", "deployment-b"]) {
|
|
const model = models.getModel("unigpt", id)!;
|
|
const stream = models.streamSimple(model, { messages: [{ role: "user", content: "Hello", timestamp: Date.now() }] });
|
|
const eventTypes: string[] = [];
|
|
for await (const event of stream) eventTypes.push(event.type);
|
|
const result = await stream.result();
|
|
assert.equal(result.stopReason, "stop", result.errorMessage);
|
|
assert.ok(eventTypes.includes("text_delta"));
|
|
assert.equal(result.content[0].type, "text");
|
|
assert.equal(result.usage.input, 12);
|
|
assert.equal(result.usage.output, 3);
|
|
}
|
|
assert.equal(requests.length, 2);
|
|
for (const [index, request] of requests.entries()) {
|
|
const id = ["deployment-a", "deployment-b"][index];
|
|
assert.equal(request.url, "/v1/responses");
|
|
assert.equal(request.headers.authorization, `Bearer ${KEY}`);
|
|
assert.equal(request.headers["deployment-id"], id);
|
|
assert.equal(request.body.model, id);
|
|
assert.equal(request.body.stream, true);
|
|
assert.equal(request.body.reasoning, undefined);
|
|
assert.equal(request.body.max_output_tokens, FALLBACK_MAX_TOKENS);
|
|
}
|
|
});
|
|
|
|
test("Responses transport preserves structured tool calls", async (t) => {
|
|
const models = await localGateway(t, ["a"], (req, res) => {
|
|
req.resume();
|
|
req.on("end", () => {
|
|
const item = { type: "function_call", id: "fc_1", call_id: "call_1", name: "ping", arguments: '{"target":"api"}', status: "completed" };
|
|
sendEvents(res, [
|
|
{ type: "response.created", response: { id: "resp_2", status: "in_progress" } },
|
|
{ type: "response.output_item.added", output_index: 0, item: { ...item, arguments: "" } },
|
|
{ type: "response.function_call_arguments.delta", item_id: "fc_1", output_index: 0, delta: item.arguments },
|
|
{ type: "response.output_item.done", output_index: 0, item },
|
|
{ type: "response.completed", response: { id: "resp_2", status: "completed", output: [item], usage: { input_tokens: 5, output_tokens: 4, total_tokens: 9 } } },
|
|
]);
|
|
});
|
|
});
|
|
const result = await models.completeSimple(models.getModel("unigpt", "a")!, {
|
|
messages: [{ role: "user", content: "Ping the API", timestamp: Date.now() }],
|
|
tools: [{ name: "ping", description: "Ping a target", parameters: { type: "object", properties: { target: { type: "string" } }, required: ["target"] } }],
|
|
});
|
|
assert.equal(result.stopReason, "toolUse", result.errorMessage);
|
|
const call = result.content.find((part) => part.type === "toolCall");
|
|
assert.ok(call && call.type === "toolCall");
|
|
assert.equal(call.name, "ping");
|
|
assert.deepEqual(call.arguments, { target: "api" });
|
|
});
|