Use overridable fetch instead of unidici's request (#1002)

When loading the schema, the `load` function used to only consider the
Content-Type header when capitalized. However, the HTTP specification
states that headers are to be treated as case-insensitive. HTTP/2
further enforces this by stating that all HTTP headers must be converted
to lowercase when encoded. Therefore, many recent implementations move
away from the conventional capitalization in favour of headers in
lowercase, such as the Nest framework.

In issue #998, it is shown that loading the schema from a live server
implemented in Nest fails when the endpoint does not include the desired
extension.

This commit fixes the issue by replacing unidici's request by fetch. The
request function returns headers as a record, where the keys are left
exactly as the headers are received, meaning that the casing of headers
affect the output. Fetch, on the other hand, returns header in a special
object whose accessor method `get` will lookup headers in a
case-insensitive way.

We took this opportunity to make the fetch implementation overridable.
This way, it is possible for the caller to swap it out for another. It
is step towards portability, as openapiTS does not run as-is in the
browser, Deno or Cloudflare Workers at the moment.

Additionally, unit tests have been added for the `load` function.

Fixes #998.

Author: yacinehmito

src/index.ts

@@ -9,7 +9,7 @@ import transformParameterObject from "./transform/parameter-object.js";
 import transformRequestBodyObject from "./transform/request-body-object.js";
 import transformResponseObject from "./transform/response-object.js";
 import transformSchemaObject from "./transform/schema-object.js";
-import { error, escObjKey, getEntries, indent } from "./utils.js";
+import { error, escObjKey, getDefaultFetch, getEntries, indent } from "./utils.js";
 export * from "./types.js"; // expose all types to consumers
 
 const EMPTY_OBJECT_RE = /^\s*\{?\s*\}?\s*$/;
@@ -67,6 +67,7 @@ async function openapiTS(
     urlCache: new Set(),
     httpHeaders: options.httpHeaders,
     httpMethod: options.httpMethod,
+    fetch: options.fetch ?? getDefaultFetch(),
   });
 
   // 1. basic validation

src/load.ts

@@ -1,5 +1,6 @@
 import type {
   ComponentsObject,
+  Fetch,
   GlobalContext,
   OpenAPI3,
   OperationObject,
@@ -15,7 +16,7 @@ import path from "node:path";
 import { Readable } from "node:stream";
 import { URL } from "node:url";
 import yaml from "js-yaml";
-import { request, type Dispatcher } from "undici";
+import type { Dispatcher } from "undici";
 import { parseRef, error, makeTSIndex, walk, isRemoteURL, isFilepath } from "./utils.js";
 
 type SchemaMap = { [id: string]: Subschema };
@@ -87,7 +88,7 @@ function parseHttpHeaders(httpHeaders: Record<string, any>): Record<string, any>
   return finalHeaders;
 }
 
-interface LoadOptions extends GlobalContext {
+export interface LoadOptions extends GlobalContext {
   /** Subschemas may be any type; this helps transform correctly */
   hint?: Subschema["hint"];
   auth?: string;
@@ -96,6 +97,7 @@ interface LoadOptions extends GlobalContext {
   urlCache: Set<string>;
   httpHeaders?: Record<string, any>;
   httpMethod?: string;
+  fetch: Fetch;
 }
 
 /** Load a schema from local path or remote URL */
@@ -104,7 +106,6 @@ export default async function load(
   options: LoadOptions
 ): Promise<{ [url: string]: Subschema }> {
   let schemaID = ".";
-
   // 1. load contents
   // 1a. URL
   if (schema instanceof URL) {
@@ -130,19 +131,20 @@ export default async function load(
           headers[k] = v;
         }
       }
-      const res = await request(schema, { method: (options.httpMethod as Dispatcher.HttpMethod) || "GET", headers });
-      const contentType = Array.isArray(res.headers["content-type"])
-        ? res.headers["content-type"][0]
-        : res.headers["content-type"];
+      const res = await options.fetch(schema, {
+        method: (options.httpMethod as Dispatcher.HttpMethod) || "GET",
+        headers,
+      });
+      const contentType = res.headers.get("content-type");
       if (ext === ".json" || (contentType && contentType.includes("json"))) {
         options.schemas[schemaID] = {
           hint,
-          schema: parseJSON(await res.body.text()),
+          schema: parseJSON(await res.text()),
         };
       } else if (ext === ".yaml" || ext === ".yml" || (contentType && contentType.includes("yaml"))) {
         options.schemas[schemaID] = {
           hint,
-          schema: parseYAML(await res.body.text()),
+          schema: parseYAML(await res.text()),
         };
       }
     }

src/types.ts

@@ -1,5 +1,6 @@
 import type { URL } from "node:url";
 import type { TransformSchemaObjectOptions } from "./transform/schema-object";
+import type { RequestInfo, RequestInit, Response } from "undici";
 
 export interface Extensable {
   [key: `x-${string}`]: any;
@@ -608,6 +609,11 @@ export interface OpenAPITSOptions {
   commentHeader?: string;
   /** (optional) inject code before schema ? */
   inject?: string;
+  /**
+   * (optional) A fetch implementation. Will default to the global fetch
+   * function if available; else, it will use unidici's fetch function.
+   */
+  fetch?: Fetch;
 }
 
 /** Subschema discriminator (note: only valid $ref types accepted here) */
@@ -637,3 +643,10 @@ export interface GlobalContext {
   silent: boolean;
   supportArrayLength: boolean;
 }
+
+// Fetch is available in the global scope starting with Node v18.
+// However, @types/node does not have it yet available.
+// GitHub issue: https://github.com/DefinitelyTyped/DefinitelyTyped/issues/60924
+// Because node's underlying implementation relies on unidici, it is safe to
+// rely on unidici's type until @types/node ships it.
+export type Fetch = (input: RequestInfo, init?: RequestInit) => Promise<Response>;

src/utils.ts

@@ -1,6 +1,8 @@
 import c from "ansi-colors";
 import { isAbsolute } from "node:path";
 import supportsColor from "supports-color";
+import { fetch as unidiciFetch } from "undici";
+import { Fetch } from "types";
 
 if (!supportsColor.stdout || supportsColor.stdout.hasBasic === false) c.enabled = false;
 
@@ -239,3 +241,11 @@ export function isRemoteURL(url: string): boolean {
 export function isFilepath(url: string): boolean {
   return url.startsWith("file://") || isAbsolute(url);
 }
+
+export function getDefaultFetch(): Fetch {
+  const globalFetch: Fetch | undefined = (globalThis as any).fetch;
+  if (typeof globalFetch === "undefined") {
+    return unidiciFetch;
+  }
+  return globalFetch;
+}

test/load.test.ts

@@ -0,0 +1,121 @@
+import internalLoad, { type LoadOptions } from "../src/load.js";
+import type { Subschema } from "../src/types";
+import { Response, fetch as unidiciFetch } from "undici";
+import { Readable } from "node:stream";
+import { dump as stringifyYaml } from "js-yaml";
+
+describe("Load", () => {
+  describe("remote schema", () => {
+    describe("in json", () => {
+      test("type deduced from extension .json", async () => {
+        const output = await load(new URL("https://example.com/openapi.json"), {
+          async fetch() {
+            return new Response(JSON.stringify(exampleSchema), {
+              headers: { "Content-Type": "text/plain" },
+            });
+          },
+        });
+        expect(output["."].schema).toEqual(exampleSchema);
+      });
+      test("type deduced from Content-Type header", async () => {
+        const output = await load(new URL("https://example.com/openapi"), {
+          async fetch() {
+            return new Response(JSON.stringify(exampleSchema), {
+              headers: { "Content-Type": "application/json" },
+            });
+          },
+        });
+        expect(output["."].schema).toEqual(exampleSchema);
+      });
+      // Regression test for https://github.com/drwpow/openapi-typescript/issues/988
+      test("type deduced from Content-Type header, in lowercase", async () => {
+        const output = await load(new URL("https://example.com/openapi"), {
+          async fetch() {
+            return new Response(JSON.stringify(exampleSchema), {
+              headers: { "content-type": "application/json" },
+            });
+          },
+        });
+        expect(output["."].schema).toEqual(exampleSchema);
+      });
+    });
+
+    describe("in yaml", () => {
+      test("type deduced from extension .yaml", async () => {
+        const output = await load(new URL("https://example.com/openapi.yaml"), {
+          async fetch() {
+            return new Response(stringifyYaml(exampleSchema), {
+              headers: { "Content-Type": "text/plain" },
+            });
+          },
+        });
+        expect(output["."].schema).toEqual(exampleSchema);
+      });
+      test("type deduced from extension .yml", async () => {
+        const output = await load(new URL("https://example.com/openapi.yml"), {
+          async fetch() {
+            return new Response(stringifyYaml(exampleSchema), {
+              headers: { "Content-Type": "text/plain" },
+            });
+          },
+        });
+        expect(output["."].schema).toEqual(exampleSchema);
+      });
+      test("type deduced from Content-Type header", async () => {
+        const output = await load(new URL("https://example.com/openapi"), {
+          async fetch() {
+            return new Response(stringifyYaml(exampleSchema), {
+              headers: { "Content-Type": "application/yaml" },
+            });
+          },
+        });
+        expect(output["."].schema).toEqual(exampleSchema);
+      });
+      // Regression test for https://github.com/drwpow/openapi-typescript/issues/988
+      test("type deduced from Content-Type header, in lowercase", async () => {
+        const output = await load(new URL("https://example.com/openapi"), {
+          async fetch() {
+            return new Response(stringifyYaml(exampleSchema), {
+              headers: { "content-type": "application/yaml" },
+            });
+          },
+        });
+        expect(output["."].schema).toEqual(exampleSchema);
+      });
+    });
+  });
+});
+
+const exampleSchema = {
+  openapi: "3.1.0",
+  paths: {
+    "/foo": {
+      get: {},
+    },
+  },
+};
+
+async function load(
+  schema: URL | Subschema | Readable,
+  options?: Partial<LoadOptions>
+): Promise<{ [url: string]: Subschema }> {
+  return internalLoad(schema, {
+    rootURL: schema as URL,
+    schemas: {},
+    urlCache: new Set(),
+    fetch: vi.fn(unidiciFetch),
+    additionalProperties: false,
+    alphabetize: false,
+    defaultNonNullable: false,
+    discriminators: {},
+    immutableTypes: false,
+    indentLv: 0,
+    operations: {},
+    pathParamsAsTypes: false,
+    postTransform: undefined,
+    silent: true,
+    supportArrayLength: false,
+    transform: undefined,
+    ...options,
+  });
+}