Ctrl K
Edits history of script submission #24008 for ' HTTP request (http)'

bun
Approved version
import { createPrivateKey } from "node:crypto"
import { METHODS } from "node:http"
import { rootCertificates } from "node:tls"

/**
 * HTTP request
 * Call any URL, or an HTTP API through an optional http resource that holds the base URL, default headers, auth (basic, bearer, OAuth 2.0 client credentials) and TLS settings (private CA, mTLS client certificate). Pick a method or type another one Bun supports (e.g. PROPFIND, PURGE). Sends a JSON, text or form body, and can follow pagination (Link header, a next URL in the body, page or offset parameters) up to max_pages. Returns status, headers and the parsed body (the collected items when paginating). A non-2xx status throws unless allow_error_status is on, which returns it instead.
 */
export async function main(
  auth: RT.Http | undefined,
  url: string,
  method:
    | "GET"
    | "POST"
    | "PUT"
    | "PATCH"
    | "DELETE"
    | "HEAD"
    | "OPTIONS"
    | undefined,
  headers: { [key: string]: string } | undefined,
  query: { [key: string]: string } | undefined,
  body: any = null,
  body_format: "json" | "text" | "form" | undefined,
  timeout_seconds: number | undefined,
  allow_error_status: boolean = false,
  pagination:
    | "none"
    | "link_header"
    | "next_url"
    | "page"
    | "offset"
    | undefined,
  next_url_path: string | undefined,
  items_path: string | undefined,
  page_param: string | undefined,
  page_start: number | undefined,
  page_size_param: string | undefined,
  page_size: number | undefined,
  max_pages: number | undefined
) {
  // Not limited to the enum: the form also takes a typed-in method. Bun's fetch
  // matches its known methods case-insensitively and silently sends GET for any other.
  const verb = (method || "GET").toUpperCase()
  if (!METHODS.includes(verb)) {
    throw new Error(
      `Bun's fetch cannot send ${verb} (it would send GET instead); it supports ${METHODS.join(", ")}`
    )
  }
  const mode = pagination || "none"
  const timeout = (timeout_seconds || 30) * 1000
  const tls = tlsOptions(auth)

  // A relative url is appended to base_url's path: /users on .../v1 is .../v1/users.
  const absolute = /^[a-z][a-z0-9+.-]*:\/\//i.test(url)
  if (!absolute && !auth?.base_url) {
    throw new Error(
      `${url} is not an absolute URL: add the scheme (https://...) or pick a resource with a base_url`
    )
  }
  const first = !absolute
    ? new URL(
        "./" + url.replace(/^\/+/, ""),
        auth!.base_url!.replace(/\/*$/, "/")
      )
    : new URL(url)
  for (const [k, v] of Object.entries(query ?? {})) {
    if (v != null && v !== "") first.searchParams.append(k, String(v))
  }

  // Header names are case-insensitive: the caller's override the resource's.
  const lower = (h: { [key: string]: any } | undefined) =>
    Object.fromEntries(
      Object.entries(h ?? {})
        .filter(([, v]) => v != null && v !== "")
        .map(([k, v]) => [k.toLowerCase(), String(v)])
    )
  const callerHeaders = lower(headers)
  const requestHeaders = { ...lower(auth?.headers), ...callerHeaders }
  if (!requestHeaders.authorization) {
    const authorization = await authHeader(auth, tls, timeout)
    if (authorization) requestHeaders.authorization = authorization
  }

  let payload: string | undefined
  if (body != null && body !== "") {
    const format = body_format || "json"
    // A string body is sent as given (already serialized); anything else as JSON,
    // whatever the format, except form fields (nested values as JSON).
    payload =
      typeof body === "string"
        ? body
        : format === "form"
          ? new URLSearchParams(
              Object.entries(body).map(([k, v]) => [
                k,
                typeof v === "string" ? v : JSON.stringify(v),
              ])
            ).toString()
          : JSON.stringify(body)
    // A Content-Type from the caller wins, then an explicit body_format, then the resource's.
    if (
      !callerHeaders["content-type"] &&
      (body_format || !requestHeaders["content-type"])
    ) {
      requestHeaders["content-type"] = {
        json: "application/json",
        text: "text/plain",
        form: "application/x-www-form-urlencoded",
      }[format]
    }
  }

  const send = async (target: URL) => {
    // Redirects are followed here, not by fetch: fetch would re-send the resource's
    // headers to another host. Off the first origin only the caller's headers go,
    // without Authorization.
    let current = target
    let verbNow = verb
    let bodyNow = payload
    let response: Response
    for (let hop = 0; ; hop++) {
      const sameOrigin = current.origin === first.origin
      const sent: Record<string, string> = sameOrigin
        ? { ...requestHeaders }
        : { ...callerHeaders }
      if (!sameOrigin) {
        delete sent.authorization
        // The body still needs its type: keep the computed one.
        if (requestHeaders["content-type"] && !sent["content-type"]) {
          sent["content-type"] = requestHeaders["content-type"]
        }
      }
      if (bodyNow === undefined) delete sent["content-type"]
      response = await fetch(current, {
        method: verbNow,
        headers: sent,
        body: bodyNow,
        redirect: "manual",
        signal: AbortSignal.timeout(timeout),
        // Nor does the client certificate, and the server is always verified there.
        tls: sameOrigin ? tls : tls && { ca: tls.ca, rejectUnauthorized: true },
      } as RequestInit)
      const location = response.headers.get("location")
      if (response.status < 300 || response.status > 399 || !location) break
      if (hop === 10) throw new Error(`Too many redirects from ${target.href}`)
      current = new URL(location, current)
      if (
        response.status === 303 ||
        ((response.status === 301 || response.status === 302) &&
          verbNow !== "GET" &&
          verbNow !== "HEAD")
      ) {
        verbNow = verbNow === "HEAD" ? "HEAD" : "GET"
        bodyNow = undefined
      }
    }
    const text = await response.text()
    const type = response.headers.get("content-type") ?? ""
    let parsed: any = text === "" ? null : text
    if (parsed !== null && /json/i.test(type)) {
      try {
        parsed = JSON.parse(text)
      } catch {}
    }
    if (!response.ok && !allow_error_status) {
      throw new Error(`${response.status} ${text}`)
    }
    return {
      status: response.status,
      headers: Object.fromEntries(response.headers),
      body: parsed,
      url: current,
    }
  }

  if (mode === "none") {
    const { url: _, ...result } = await send(first)
    return result
  }
  if (page_size && !page_size_param && (mode === "page" || mode === "offset")) {
    throw new Error(
      "page_size needs page_size_param, the query parameter that carries it"
    )
  }

  if (mode === "next_url" && !next_url_path) {
    throw new Error(
      "next_url pagination needs next_url_path, the path of the next page's URL in the response"
    )
  }
  // Pagination: collect the items of each page (or each page's body when there is no items list).
  const limit = max_pages || 10
  const offsetMode = mode === "offset"
  let position = page_start ?? (offsetMode ? 0 : 1)
  const items: any[] = []
  let target: URL | undefined = first
  let last: Awaited<ReturnType<typeof send>> | undefined
  let pages = 0
  while (target && pages < limit) {
    if (mode === "page" || offsetMode) {
      target.searchParams.set(
        page_param || (offsetMode ? "offset" : "page"),
        String(position)
      )
      if (page_size_param && page_size) {
        target.searchParams.set(page_size_param, String(page_size))
      }
    }
    last = await send(target)
    pages++
    // Only with allow_error_status: stop there and return the error response's body.
    if (last.status >= 300) {
      const { url: _, ...error } = last
      return {
        ...error,
        body: items,
        error_body: last.body,
        pages,
        has_more: true,
      }
    }
    const pageItems = items_path ? pick(last.body, items_path) : last.body
    // Some APIs drop or null the item list past the last page: that ends it too.
    const list = Array.isArray(pageItems)
      ? pageItems
      : pages > 1 && pageItems == null && (mode === "page" || offsetMode)
        ? []
        : undefined
    if ((mode === "page" || offsetMode) && !list) {
      throw new Error(
        `Page ${pages} has no item list${items_path ? ` at ${items_path}` : ""}: set items_path to the array of items in the response`
      )
    }
    if (list) items.push(...list)
    else items.push(last.body)

    let next: string | undefined
    if (mode === "link_header") next = nextLink(last.headers.link)
    else if (mode === "next_url") {
      const value = pick(last.body, next_url_path!)
      next = typeof value === "string" && value !== "" ? value : undefined
    } else if (list!.length > 0) {
      // Stop on an empty page, not a short one: APIs cap the page size silently.
      position += offsetMode ? list!.length : 1
      next = target.href
    }
    if (!next) {
      target = undefined
      break
    }
    // Resolved against where the page actually came from, after redirects.
    const following: URL = new URL(next, last.url)
    // Credentials only go to the origin of the first request.
    if (following.origin !== first.origin) {
      throw new Error(
        `Next page ${following.href} is on another origin than ${first.origin}`
      )
    }
    target = following
  }
  return {
    status: last!.status,
    headers: last!.headers,
    body: items,
    pages,
    has_more: target != null,
  }
}

function pick(value: any, path: string) {
  return path
    .split(".")
    .reduce((v, key) => (v == null ? undefined : v[key]), value)
}

function nextLink(link: string | undefined) {
  // Link: <https://api.example.com/items?page=2>; rel="next", <...>; rel="last"
  for (const part of link?.split(/,(?=\s*<)/) ?? []) {
    const rel = part.match(/;\s*rel="?([^";]*)/i)?.[1]
    if (rel?.toLowerCase().split(/\s+/).includes("next")) {
      return part.match(/<([^>]*)>/)?.[1]
    }
  }
  return undefined
}

function tlsOptions(auth: RT.Http | undefined) {
  const t = auth?.tls
  if (!t) return undefined
  return {
    // A custom ca replaces the default list: keep the system CAs alongside it.
    ca: t.ca_certificate ? [...rootCertificates, t.ca_certificate] : undefined,
    cert: t.client_certificate || undefined,
    // Decrypt here: the key is handed to the TLS client as plain PEM.
    key: t.client_key
      ? t.client_key_password
        ? createPrivateKey({
            key: t.client_key,
            passphrase: t.client_key_password,
          })
            .export({ type: "pkcs8", format: "pem" })
            .toString()
        : t.client_key
      : undefined,
    rejectUnauthorized: t.verify !== false,
  }
}

async function authHeader(
  auth: RT.Http | undefined,
  tls: ReturnType<typeof tlsOptions>,
  timeout: number
) {
  const a = auth?.auth
  if (!a || a.label === "NONE") return undefined
  if (a.label === "BASIC")
    return `Basic ${base64(`${a.username}:${a.password}`)}`
  if (a.label === "BEARER") return `Bearer ${a.token}`
  // OAUTH2_CLIENT_CREDENTIALS: one token per run (RFC 6749 section 4.4).
  const form = new URLSearchParams({ grant_type: "client_credentials" })
  if (a.scope) form.set("scope", a.scope)
  if (a.audience) form.set("audience", a.audience)
  const tokenHeaders: Record<string, string> = {
    "Content-Type": "application/x-www-form-urlencoded",
    Accept: "application/json",
  }
  if (a.client_auth === "body") {
    form.set("client_id", a.client_id)
    form.set("client_secret", a.client_secret)
  } else {
    const enc = (s: string) => encodeURIComponent(s)
    tokenHeaders.Authorization = `Basic ${base64(`${enc(a.client_id)}:${enc(a.client_secret)}`)}`
  }
  const response = await fetch(a.token_url, {
    method: "POST",
    headers: tokenHeaders,
    body: form,
    signal: AbortSignal.timeout(timeout),
    // verify off is for the API server only: the IdP receiving the secret is always verified.
    tls: tls && { ...tls, rejectUnauthorized: true },
  } as RequestInit)
  if (!response.ok) {
    throw new Error(
      `Token request: ${response.status} ${await response.text()}`
    )
  }
  const text = await response.text()
  let access_token: unknown
  try {
    access_token = JSON.parse(text).access_token
  } catch {}
  if (typeof access_token !== "string" || !access_token) {
    throw new Error(`Token request: no access_token in ${text}`)
  }
  return `Bearer ${access_token}`
}

function base64(s: string) {
  return Buffer.from(s).toString("base64")
}
Submitted by hugo989 8 hours ago
← Back to the list of replies