Scrape Elements

//native

/**

 * Scrape Elements

 * Load a page (url or raw html) and return the text, html and attributes of the elements matching each CSS selector. Extra fields go in options (gotoOptions, viewport, waitForSelector, userAgent, ...).

 */

export async function main(

  auth: RT.Cloudflare,

  url: string | undefined,

  html: string | undefined,

  selectors: string[],

  options: { [key: string]: any } | undefined

) {

  if (!url && !html) throw new Error("Either url or html is required")

  const response = await fetch(

    `https://api.cloudflare.com/client/v4/accounts/${auth.account_id}/browser-rendering/scrape`,

    {

      method: "POST",

      headers: {

        Authorization: `Bearer ${auth.token}`,

        "Content-Type": "application/json",

      },

      body: JSON.stringify({

        url: url || undefined,

        html: html || undefined,

        elements: selectors.map((selector) => ({ selector })),

        ...options,

      }),

    }

  )

  if (!response.ok) {

    throw new Error(`${response.status} ${await response.text()}`)

  }

  return await response.json()

}