HTTP request

import { createPrivateKey } from "node:crypto"

import { METHODS } from "node:http"

import { rootCertificates } from "node:tls"

/**

 * HTTP request

 * Call any URL, or an HTTP API through an optional http resource that holds the base URL, default headers, auth (basic, bearer, OAuth 2.0 client credentials) and TLS settings (private CA, mTLS client certificate). Pick a method or type another one Bun supports (e.g. PROPFIND, PURGE). Sends a JSON, text or form body, and can follow pagination (Link header, a next URL in the body, page or offset parameters) up to max_pages. Returns status, headers and the parsed body (the collected items when paginating). A non-2xx status throws unless allow_error_status is on, which returns it instead.

 */

export async function main(

  auth: RT.Http | undefined,

  url: string,

  method:

    | "GET"

    | "POST"

    | "PUT"

    | "PATCH"

    | "DELETE"

    | "HEAD"

    | "OPTIONS"

    | undefined,

  headers: { [key: string]: string } | undefined,

  query: { [key: string]: string } | undefined,

  body: any = null,

  body_format: "json" | "text" | "form" | undefined,

  timeout_seconds: number | undefined,

  allow_error_status: boolean = false,

  pagination:

    | "none"

    | "link_header"

    | "next_url"

    | "page"

    | "offset"

    | undefined,

  next_url_path: string | undefined,

  items_path: string | undefined,

  page_param: string | undefined,

  page_start: number | undefined,

  page_size_param: string | undefined,

  page_size: number | undefined,

  max_pages: number | undefined

) {

  // Not limited to the enum: the form also takes a typed-in method. Bun's fetch

  // matches its known methods case-insensitively and silently sends GET for any other.

  const verb = (method || "GET").toUpperCase()

  if (!METHODS.includes(verb)) {

    throw new Error(

      `Bun's fetch cannot send ${verb} (it would send GET instead); it supports ${METHODS.join(", ")}`

    )

  }

  const mode = pagination || "none"

  const timeout = (timeout_seconds || 30) * 1000

  const tls = tlsOptions(auth)

  // A relative url is appended to base_url's path: /users on .../v1 is .../v1/users.

  const absolute = /^[a-z][a-z0-9+.-]*:\/\//i.test(url)

  if (!absolute && !auth?.base_url) {

    throw new Error(

      `${url} is not an absolute URL: add the scheme (https://...) or pick a resource with a base_url`

    )

  }

  const first = !absolute

    ? new URL(

        "./" + url.replace(/^\/+/, ""),

        auth!.base_url!.replace(/\/*$/, "/")

      )

    : new URL(url)

  for (const [k, v] of Object.entries(query ?? {})) {

    if (v != null && v !== "") first.searchParams.append(k, String(v))

  }

  // Header names are case-insensitive: the caller's override the resource's.

  const lower = (h: { [key: string]: any } | undefined) =>

    Object.fromEntries(

      Object.entries(h ?? {})

        .filter(([, v]) => v != null && v !== "")

        .map(([k, v]) => [k.toLowerCase(), String(v)])

    )

  const callerHeaders = lower(headers)

  const requestHeaders = { ...lower(auth?.headers), ...callerHeaders }

  if (!requestHeaders.authorization) {

    const authorization = await authHeader(auth, tls, timeout)

    if (authorization) requestHeaders.authorization = authorization

  }

  let payload: string | undefined

  if (body != null && body !== "") {

    const format = body_format || "json"

    // A string body is sent as given (already serialized); anything else as JSON,

    // whatever the format, except form fields (nested values as JSON).

    payload =

      typeof body === "string"

        ? body

        : format === "form"

          ? new URLSearchParams(

              Object.entries(body).map(([k, v]) => [

                k,

                typeof v === "string" ? v : JSON.stringify(v),

              ])

            ).toString()

          : JSON.stringify(body)

    // A Content-Type from the caller wins, then an explicit body_format, then the resource's.

    if (

      !callerHeaders["content-type"] &&

      (body_format || !requestHeaders["content-type"])

    ) {

      requestHeaders["content-type"] = {

        json: "application/json",

        text: "text/plain",

        form: "application/x-www-form-urlencoded",

      }[format]

    }

  }

  const send = async (target: URL) => {

    // Redirects are followed here, not by fetch: fetch would re-send the resource's

    // headers to another host. Off the first origin only the caller's headers go,

    // without Authorization.

    let current = target

    let verbNow = verb

    let bodyNow = payload

    let response: Response

    for (let hop = 0; ; hop++) {

      const sameOrigin = current.origin === first.origin

      const sent: Record<string, string> = sameOrigin

        ? { ...requestHeaders }

        : { ...callerHeaders }

      if (!sameOrigin) {

        delete sent.authorization

        // The body still needs its type: keep the computed one.

        if (requestHeaders["content-type"] && !sent["content-type"]) {

          sent["content-type"] = requestHeaders["content-type"]

        }

      }

      if (bodyNow === undefined) delete sent["content-type"]

      response = await fetch(current, {

        method: verbNow,

        headers: sent,

        body: bodyNow,

        redirect: "manual",

        signal: AbortSignal.timeout(timeout),

        // Nor does the client certificate, and the server is always verified there.

        tls: sameOrigin ? tls : tls && { ca: tls.ca, rejectUnauthorized: true },

      } as RequestInit)

      const location = response.headers.get("location")

      if (response.status < 300 || response.status > 399 || !location) break

      if (hop === 10) throw new Error(`Too many redirects from ${target.href}`)

      current = new URL(location, current)

      if (

        response.status === 303 ||

        ((response.status === 301 || response.status === 302) &&

          verbNow !== "GET" &&

          verbNow !== "HEAD")

      ) {

        verbNow = verbNow === "HEAD" ? "HEAD" : "GET"

        bodyNow = undefined

      }

    }

    const text = await response.text()

    const type = response.headers.get("content-type") ?? ""

    let parsed: any = text === "" ? null : text

    if (parsed !== null && /json/i.test(type)) {

      try {

        parsed = JSON.parse(text)

      } catch {}

    }

    if (!response.ok && !allow_error_status) {

      throw new Error(`${response.status} ${text}`)

    }

    return {

      status: response.status,

      headers: Object.fromEntries(response.headers),

      body: parsed,

      url: current,

    }

  }

  if (mode === "none") {

    const { url: _, ...result } = await send(first)

    return result

  }

  if (page_size && !page_size_param && (mode === "page" || mode === "offset")) {

    throw new Error(

      "page_size needs page_size_param, the query parameter that carries it"

    )

  }

  if (mode === "next_url" && !next_url_path) {

    throw new Error(

      "next_url pagination needs next_url_path, the path of the next page's URL in the response"

    )

  }

  // Pagination: collect the items of each page (or each page's body when there is no items list).

  const limit = max_pages || 10

  const offsetMode = mode === "offset"

  let position = page_start ?? (offsetMode ? 0 : 1)

  const items: any[] = []

  let target: URL | undefined = first

  let last: Awaited<ReturnType<typeof send>> | undefined

  let pages = 0

  while (target && pages < limit) {

    if (mode === "page" || offsetMode) {

      target.searchParams.set(

        page_param || (offsetMode ? "offset" : "page"),

        String(position)

      )

      if (page_size_param && page_size) {

        target.searchParams.set(page_size_param, String(page_size))

      }

    }

    last = await send(target)

    pages++

    // Only with allow_error_status: stop there and return the error response's body.

    if (last.status >= 300) {

      const { url: _, ...error } = last

      return {

        ...error,

        body: items,

        error_body: last.body,

        pages,

        has_more: true,

      }

    }

    const pageItems = items_path ? pick(last.body, items_path) : last.body

    // Some APIs drop or null the item list past the last page: that ends it too.

    const list = Array.isArray(pageItems)

      ? pageItems

      : pages > 1 && pageItems == null && (mode === "page" || offsetMode)

        ? []

        : undefined

    if ((mode === "page" || offsetMode) && !list) {

      throw new Error(

        `Page ${pages} has no item list${items_path ? ` at ${items_path}` : ""}: set items_path to the array of items in the response`

      )

    }

    if (list) items.push(...list)

    else items.push(last.body)

    let next: string | undefined

    if (mode === "link_header") next = nextLink(last.headers.link)

    else if (mode === "next_url") {

      const value = pick(last.body, next_url_path!)

      next = typeof value === "string" && value !== "" ? value : undefined

    } else if (list!.length > 0) {

      // Stop on an empty page, not a short one: APIs cap the page size silently.

      position += offsetMode ? list!.length : 1

      next = target.href

    }

    if (!next) {

      target = undefined

      break

    }

    // Resolved against where the page actually came from, after redirects.

    const following: URL = new URL(next, last.url)

    // Credentials only go to the origin of the first request.

    if (following.origin !== first.origin) {

      throw new Error(

        `Next page ${following.href} is on another origin than ${first.origin}`

      )

    }

    target = following

  }

  return {

    status: last!.status,

    headers: last!.headers,

    body: items,

    pages,

    has_more: target != null,

  }

}

function pick(value: any, path: string) {

  return path

    .split(".")

    .reduce((v, key) => (v == null ? undefined : v[key]), value)

}

function nextLink(link: string | undefined) {

  // Link: <https://api.example.com/items?page=2>; rel="next", <...>; rel="last"

  for (const part of link?.split(/,(?=\s*<)/) ?? []) {

    const rel = part.match(/;\s*rel="?([^";]*)/i)?.[1]

    if (rel?.toLowerCase().split(/\s+/).includes("next")) {

      return part.match(/<([^>]*)>/)?.[1]

    }

  }

  return undefined

}

function tlsOptions(auth: RT.Http | undefined) {

  const t = auth?.tls

  if (!t) return undefined

  return {

    // A custom ca replaces the default list: keep the system CAs alongside it.

    ca: t.ca_certificate ? [...rootCertificates, t.ca_certificate] : undefined,

    cert: t.client_certificate || undefined,

    // Decrypt here: the key is handed to the TLS client as plain PEM.

    key: t.client_key

      ? t.client_key_password

        ? createPrivateKey({

            key: t.client_key,

            passphrase: t.client_key_password,

          })

            .export({ type: "pkcs8", format: "pem" })

            .toString()

        : t.client_key

      : undefined,

    rejectUnauthorized: t.verify !== false,

  }

}

async function authHeader(

  auth: RT.Http | undefined,

  tls: ReturnType<typeof tlsOptions>,

  timeout: number

) {

  const a = auth?.auth

  if (!a || a.label === "NONE") return undefined

  if (a.label === "BASIC")

    return `Basic ${base64(`${a.username}:${a.password}`)}`

  if (a.label === "BEARER") return `Bearer ${a.token}`

  // OAUTH2_CLIENT_CREDENTIALS: one token per run (RFC 6749 section 4.4).

  const form = new URLSearchParams({ grant_type: "client_credentials" })

  if (a.scope) form.set("scope", a.scope)

  if (a.audience) form.set("audience", a.audience)

  const tokenHeaders: Record<string, string> = {

    "Content-Type": "application/x-www-form-urlencoded",

    Accept: "application/json",

  }

  if (a.client_auth === "body") {

    form.set("client_id", a.client_id)

    form.set("client_secret", a.client_secret)

  } else {

    const enc = (s: string) => encodeURIComponent(s)

    tokenHeaders.Authorization = `Basic ${base64(`${enc(a.client_id)}:${enc(a.client_secret)}`)}`

  }

  const response = await fetch(a.token_url, {

    method: "POST",

    headers: tokenHeaders,

    body: form,

    signal: AbortSignal.timeout(timeout),

    // verify off is for the API server only: the IdP receiving the secret is always verified.

    tls: tls && { ...tls, rejectUnauthorized: true },

  } as RequestInit)

  if (!response.ok) {

    throw new Error(

      `Token request: ${response.status} ${await response.text()}`

    )

  }

  const text = await response.text()

  let access_token: unknown

  try {

    access_token = JSON.parse(text).access_token

  } catch {}

  if (typeof access_token !== "string" || !access_token) {

    throw new Error(`Token request: no access_token in ${text}`)

  }

  return `Bearer ${access_token}`

}

function base64(s: string) {

  return Buffer.from(s).toString("base64")

}