import assert from "node:assert/strict"; import { once } from "node:events"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import { test, type TestContext } from "node:test"; import { createModels, InMemoryCredentialStore, InMemoryModelsStore, type AuthEvent, type AuthInteraction, } from "@earendil-works/pi-ai"; import { BASE_URL, createUnigptProvider, FALLBACK_CONTEXT_WINDOW, FALLBACK_MAX_TOKENS, fetchCatalog, lookupCatalogMetadata, makeModel, parseCatalog, type ModelMetadata, } from "../src/provider.ts"; const KEY = "test-key-not-a-real-secret"; const signal = () => new AbortController().signal; const catalog = (...ids: string[]) => Response.json({ data: ids.map((id) => ({ id })) }); function interaction(key = KEY, events: AuthEvent[] = []): AuthInteraction { return { prompt: async () => key, notify: (event) => events.push(event) }; } const ids = async (...args: Parameters) => (await fetchCatalog(...args)).ids; async function runtime(fetchImpl: typeof fetch, withCredential = true) { const credentials = new InMemoryCredentialStore(); if (withCredential) { await credentials.modify("unigpt", async () => ({ type: "api_key", key: KEY })); } const modelsStore = new InMemoryModelsStore(); const models = createModels({ credentials, modelsStore }); const provider = createUnigptProvider({ fetch: fetchImpl }); models.setProvider(provider); return { models, provider, credentials, modelsStore }; } test("parses IDs, deduplicates in catalog order, and permits an empty catalog", () => { assert.deepEqual(parseCatalog({ data: [{ id: "second" }, { id: "first" }, { id: "second" }] }), { ids: ["second", "first"], skipped: 0 }); assert.deepEqual(parseCatalog({ data: [] }), { ids: [], skipped: 0 }); }); test("skips unusable entries but rejects malformed or entirely unusable catalogs", () => { assert.deepEqual( parseCatalog({ data: [{ id: "good" }, null, {}, { id: 42 }, { id: "" }, { id: " padded " }, { id: "header\ninjection" }] }), { ids: ["good"], skipped: 6 }, ); for (const payload of [null, [], {}, { data: {} }, { data: [null] }, { data: [{ id: "a\r\nb" }] }]) { assert.throws(() => parseCatalog(payload), /invalid|without any valid/); } }); const known: Record = { "model-a": { name: "Model A", input: ["text", "image"], contextWindow: 272_000, maxTokens: 128_000 }, }; const lookup = (id: string) => known[id]; test("known deployments take pi's catalog metadata; each gets its own header", () => { const a = makeModel("model-a", BASE_URL, lookup); const b = makeModel("model-b", BASE_URL, lookup); assert.deepEqual(a.headers, { "deployment-id": "model-a" }); assert.deepEqual(b.headers, { "deployment-id": "model-b" }); assert.equal(a.provider, "unigpt"); assert.equal(a.api, "openai-responses"); assert.equal(a.name, "Model A"); assert.deepEqual(a.input, ["text", "image"]); assert.equal(a.contextWindow, 272_000); assert.equal(a.maxTokens, 128_000); // UniGPT ignores reasoning.effort, so no deployment advertises thinking controls. assert.equal(a.reasoning, false); assert.deepEqual(a.cost, { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }); assert.equal(a.promptCache, undefined); }); test("unknown deployments fall back to pi's custom-model defaults", () => { const b = makeModel("model-b", BASE_URL, lookup); assert.equal(b.name, "model-b"); assert.deepEqual(b.input, ["text"]); assert.equal(b.contextWindow, FALLBACK_CONTEXT_WINDOW); assert.equal(b.maxTokens, FALLBACK_MAX_TOKENS); assert.equal(b.reasoning, false); }); test("catalog lookup uses pi's bundled OpenAI/Azure entries and takes the conservative limit", () => { const gpt4o = lookupCatalogMetadata("gpt-4o"); assert.ok(gpt4o, "pi's catalog should know gpt-4o"); assert.deepEqual(gpt4o.input, ["text", "image"]); assert.ok(gpt4o.contextWindow >= 100_000 && gpt4o.maxTokens >= 4_096); assert.equal(lookupCatalogMetadata("definitely-not-a-model"), undefined); }); test("discovery uses bearer auth, no deployment header, a timeout signal, and no redirects", async () => { const fakeFetch: typeof fetch = async (url, options) => { assert.equal(url, BASE_URL + "/models"); const headers = new Headers(options?.headers); assert.equal(headers.get("Authorization"), `Bearer ${KEY}`); assert.equal(headers.get("deployment-id"), null); assert.equal(headers.get("Accept"), "application/json"); assert.equal(options?.redirect, "error"); assert.ok(options?.signal instanceof AbortSignal); return catalog("model-a"); }; assert.deepEqual(await ids(BASE_URL + "///", KEY, signal(), fakeFetch), ["model-a"]); }); test("unconfigured providers do not fetch or become available", async () => { let calls = 0; const { models } = await runtime(async () => { calls++; return catalog("a"); }, false); assert.deepEqual(await models.getAvailable("unigpt"), []); assert.equal(await models.getAuth("unigpt"), undefined); assert.equal((await models.refresh()).errors.size, 0); assert.equal(calls, 0); }); test("native login masks input, validates access, and lets pi persist the credential", async () => { let calls = 0; const events: AuthEvent[] = []; const { models, credentials } = await runtime(async () => { calls++; return catalog("a", "b"); }, false); await models.login("unigpt", "api_key", { prompt: async (prompt) => { assert.equal(prompt.type, "secret"); return ` ${KEY} `; }, notify: (event) => events.push(event), }); assert.equal(calls, 1); assert.ok(!JSON.stringify(events).includes(KEY)); assert.ok(events.some((e) => e.type === "info" && e.links?.some((link) => link.url.startsWith("https://unigpt.uni-graz.at")))); assert.ok(events.some((e) => e.type === "info" && /2 UniGPT models available/.test(e.message))); assert.deepEqual(await credentials.read("unigpt"), { type: "api_key", key: KEY }); assert.equal((await models.refresh()).errors.size, 0); assert.deepEqual(models.getModels("unigpt").map((m) => m.id), ["a", "b"]); assert.equal((await models.getAvailable("unigpt")).length, 2); await models.logout("unigpt"); assert.equal(await credentials.read("unigpt"), undefined); assert.deepEqual(await models.getAvailable("unigpt"), []); }); test("empty and rejected API keys never overwrite a saved credential", async () => { let calls = 0; const { models, credentials } = await runtime(async () => { calls++; return new Response("private upstream body", { status: 401 }); }); await assert.rejects(models.login("unigpt", "api_key", interaction(" ")), /required/); assert.equal(calls, 0); await assert.rejects(models.login("unigpt", "api_key", interaction("invalid-new-key")), /401/); assert.deepEqual(await credentials.read("unigpt"), { type: "api_key", key: KEY }); }); test("login with an empty catalog still saves the key but explains the empty model list", async () => { const events: AuthEvent[] = []; const { models, credentials } = await runtime(async () => catalog(), false); await models.login("unigpt", "api_key", interaction(KEY, events)); assert.deepEqual(await credentials.read("unigpt"), { type: "api_key", key: KEY }); assert.ok(events.some((e) => e.type === "info" && /lists no models/.test(e.message))); }); test("cancelled login never saves a key or sends validation traffic", async () => { const controller = new AbortController(); let calls = 0; const { models, credentials } = await runtime(async () => { calls++; return catalog("a"); }, false); await assert.rejects(models.login("unigpt", "api_key", { signal: controller.signal, prompt: async () => { controller.abort(); return KEY; }, notify: () => {}, }), { name: "AbortError" }); assert.equal(calls, 0); assert.equal(await credentials.read("unigpt"), undefined); }); test("discovery errors are useful but never echo secrets or server error bodies", async () => { for (const status of [401, 403, 404, 429, 500]) { await assert.rejects(fetchCatalog(BASE_URL, KEY, signal(), async () => new Response(KEY, { status })), (error: Error) => { assert.match(error.message, new RegExp(String(status))); assert.ok(!error.message.includes(KEY)); return true; }); } await assert.rejects(fetchCatalog(BASE_URL, KEY, signal(), async () => { throw new Error(KEY); }), (error: Error) => { assert.match(error.message, /Could not reach/); assert.ok(!error.message.includes(KEY)); return true; }); await assert.rejects(fetchCatalog(BASE_URL, KEY, signal(), async () => new Response("not-json")), /not valid JSON/); }); test("rate limits surface Retry-After", async () => { const limited = async () => new Response(null, { status: 429, headers: { "Retry-After": "30" } }); await assert.rejects(fetchCatalog(BASE_URL, KEY, signal(), limited), /rate-limiting.*Retry in 30s/); }); test("a stalled endpoint produces a readable timeout, not an AbortError", async () => { const stalled: typeof fetch = (_url, options) => new Promise((_resolve, reject) => { options!.signal!.addEventListener("abort", () => reject(options!.signal!.reason), { once: true }); }); await assert.rejects(fetchCatalog(BASE_URL, KEY, signal(), stalled, 20), (error: Error) => { assert.equal(error.name, "Error"); assert.match(error.message, /did not respond within/); return true; }); }); test("discovery propagates cancellation to fetch", async () => { const controller = new AbortController(); let started!: () => void; const ready = new Promise((resolve) => { started = resolve; }); const promise = fetchCatalog(BASE_URL, KEY, controller.signal, async (_url: unknown, options?: RequestInit) => { started(); return new Promise((_resolve, reject) => { options!.signal!.addEventListener("abort", () => reject(options!.signal!.reason), { once: true }); }); }); await ready; controller.abort(); await assert.rejects(promise, { name: "AbortError" }); }); test("refresh persists catalog metadata only, restores offline, and keeps cache on failure", async () => { let fail = false; const { models, provider, credentials, modelsStore } = await runtime(async () => { if (fail) return new Response("unavailable", { status: 503 }); return catalog("a", "b"); }); assert.equal((await models.refresh()).errors.size, 0); const stored = await modelsStore.read("unigpt"); assert.ok(stored); assert.ok(!JSON.stringify(stored).includes(KEY)); assert.deepEqual(stored.models.map((m) => m.id), ["a", "b"]); fail = true; assert.equal((await models.refresh()).errors.size, 1); assert.deepEqual(provider.getModels().map((m) => m.id), ["a", "b"]); let calls = 0; const offline = createModels({ credentials, modelsStore }); offline.setProvider(createUnigptProvider({ fetch: async () => { calls++; throw new Error("no network"); } })); assert.equal((await offline.refresh({ allowNetwork: false })).errors.size, 0); assert.equal(calls, 0); assert.deepEqual(offline.getModels("unigpt").map((m) => m.id), ["a", "b"]); }); test("malformed refresh preserves the previous catalog; partial and valid refreshes replace it", async () => { let response = () => catalog("a", "b"); const { models } = await runtime(async () => response()); await models.refresh(); response = () => Response.json({ data: [{}, null] }); assert.equal((await models.refresh()).errors.size, 1); assert.deepEqual(models.getModels("unigpt").map((m) => m.id), ["a", "b"]); response = () => Response.json({ data: [{ id: "a" }, {}] }); assert.equal((await models.refresh()).errors.size, 0); assert.deepEqual(models.getModels("unigpt").map((m) => m.id), ["a"]); response = () => catalog("b", "c"); await models.refresh(); assert.deepEqual(models.getModels("unigpt").map((m) => m.id), ["b", "c"]); response = () => catalog(); await models.refresh(); assert.deepEqual(models.getModels("unigpt"), []); }); test("discovery honors the effective base URL supplied by pi", async () => { const alternate = "https://gateway.example.test/v1"; const provider = createUnigptProvider({ fetch: async (url) => { assert.equal(url, alternate + "/models"); return catalog("a"); } }); await provider.refreshModels!({ credential: { type: "api_key", key: KEY }, baseUrl: alternate, signal: signal(), allowNetwork: true, publish: async (publication) => { publication.update?.(); return true; }, }); assert.equal(provider.getModels()[0].baseUrl, alternate); }); /** A local Responses endpoint plus a configured runtime whose catalog lists `ids`. */ async function localGateway(t: TestContext, ids: string[], handler: (req: IncomingMessage, res: ServerResponse) => void) { const server = createServer(handler); server.listen(0, "127.0.0.1"); await once(server, "listening"); t.after(() => new Promise((resolve) => { server.closeAllConnections(); server.close(() => resolve()); })); const address = server.address(); assert.ok(address && typeof address !== "string"); const credentials = new InMemoryCredentialStore(); await credentials.modify("unigpt", async () => ({ type: "api_key", key: KEY })); const models = createModels({ credentials }); models.setProvider(createUnigptProvider({ baseUrl: `http://127.0.0.1:${address.port}/v1`, fetch: async () => catalog(...ids), lookup: () => undefined, })); await models.refresh(); return models; } function sendEvents(res: ServerResponse, events: unknown[]) { res.writeHead(200, { "Content-Type": "text/event-stream" }); for (const event of events) res.write(`data: ${JSON.stringify(event)}\n\n`); res.end("data: [DONE]\n\n"); } test("real Responses transport sends deployment/auth headers, streams text, and records usage", async (t) => { const requests: { url?: string; headers: IncomingMessage["headers"]; body: Record }[] = []; const models = await localGateway(t, ["deployment-a", "deployment-b"], (req, res) => { let body = ""; req.on("data", (chunk) => { body += chunk; }); req.on("end", () => { requests.push({ url: req.url, headers: req.headers, body: JSON.parse(body) }); const item = { id: "msg_1", type: "message", role: "assistant", status: "completed", content: [{ type: "output_text", text: "Hello!", annotations: [] }] }; sendEvents(res, [ { type: "response.created", response: { id: "resp_1", status: "in_progress" } }, { type: "response.output_item.added", output_index: 0, item: { ...item, content: [] } }, { type: "response.output_text.delta", item_id: "msg_1", output_index: 0, content_index: 0, delta: "Hello!" }, { type: "response.output_item.done", output_index: 0, item }, { type: "response.completed", response: { id: "resp_1", status: "completed", output: [item], usage: { input_tokens: 12, output_tokens: 3, total_tokens: 15 } } }, ]); }); }); for (const id of ["deployment-a", "deployment-b"]) { const model = models.getModel("unigpt", id)!; const stream = models.streamSimple(model, { messages: [{ role: "user", content: "Hello", timestamp: Date.now() }] }); const eventTypes: string[] = []; for await (const event of stream) eventTypes.push(event.type); const result = await stream.result(); assert.equal(result.stopReason, "stop", result.errorMessage); assert.ok(eventTypes.includes("text_delta")); assert.equal(result.content[0].type, "text"); assert.equal(result.usage.input, 12); assert.equal(result.usage.output, 3); } assert.equal(requests.length, 2); for (const [index, request] of requests.entries()) { const id = ["deployment-a", "deployment-b"][index]; assert.equal(request.url, "/v1/responses"); assert.equal(request.headers.authorization, `Bearer ${KEY}`); assert.equal(request.headers["deployment-id"], id); assert.equal(request.body.model, id); assert.equal(request.body.stream, true); assert.equal(request.body.reasoning, undefined); assert.equal(request.body.max_output_tokens, FALLBACK_MAX_TOKENS); } }); test("Responses transport preserves structured tool calls", async (t) => { const models = await localGateway(t, ["a"], (req, res) => { req.resume(); req.on("end", () => { const item = { type: "function_call", id: "fc_1", call_id: "call_1", name: "ping", arguments: '{"target":"api"}', status: "completed" }; sendEvents(res, [ { type: "response.created", response: { id: "resp_2", status: "in_progress" } }, { type: "response.output_item.added", output_index: 0, item: { ...item, arguments: "" } }, { type: "response.function_call_arguments.delta", item_id: "fc_1", output_index: 0, delta: item.arguments }, { type: "response.output_item.done", output_index: 0, item }, { type: "response.completed", response: { id: "resp_2", status: "completed", output: [item], usage: { input_tokens: 5, output_tokens: 4, total_tokens: 9 } } }, ]); }); }); const result = await models.completeSimple(models.getModel("unigpt", "a")!, { messages: [{ role: "user", content: "Ping the API", timestamp: Date.now() }], tools: [{ name: "ping", description: "Ping a target", parameters: { type: "object", properties: { target: { type: "string" } }, required: ["target"] } }], }); assert.equal(result.stopReason, "toolUse", result.errorMessage); const call = result.content.find((part) => part.type === "toolCall"); assert.ok(call && call.type === "toolCall"); assert.equal(call.name, "ping"); assert.deepEqual(call.arguments, { target: "api" }); });