Ctrl K

Get an answer to a question about a given web page

Published Nov 5, 2024

Returns the answer in plain text. Proxies and Chromium JavaScript rendering are used for page retrieval and processing, then the answer is extracted using an LLM model.

Script webscrapingai Verified

Use in Windmill

The script

Submitted by hugo697 Bun

Verified 581 days ago

All edits

Permalink

//native
type Webscrapingai = {
	apiKey: string
}
/**
 * Get an answer to a question about a given web page
 * Returns the answer in plain text. Proxies and Chromium JavaScript rendering are used for page retrieval and processing, then the answer is extracted using an LLM model.
 */
export async function main(
	auth: Webscrapingai,
	url: string | undefined,
	question: string | undefined,
	context_limit: '4000' | '8000' | '16000' | undefined,
	response_tokens: string | undefined,
	on_context_limit: 'truncate' | 'error' | undefined,
	headers: any,
	timeout: string | undefined,
	js: string | undefined,
	js_timeout: string | undefined,
	wait_for: string | undefined,
	proxy: 'datacenter' | 'residential' | undefined,
	country: 'us' | 'gb' | 'de' | 'it' | 'fr' | 'ca' | 'es' | 'ru' | 'jp' | 'kr' | 'in' | undefined,
	custom_proxy: string | undefined,
	device: 'desktop' | 'mobile' | 'tablet' | undefined,
	error_on_404: string | undefined,
	error_on_redirect: string | undefined,
	js_script: string | undefined,
	format: 'json' | 'text' | undefined
) {
	const url_ = new URL(`https://api.webscraping.ai/ai/question`)

	url_.searchParams.append('api_key', auth.apiKey)

	for (const [k, v] of [
		['url', url],
		['question', question],
		['context_limit', context_limit],
		['response_tokens', response_tokens],
		['on_context_limit', on_context_limit],
		['timeout', timeout],
		['js', js],
		['js_timeout', js_timeout],
		['wait_for', wait_for],
		['proxy', proxy],
		['country', country],
		['custom_proxy', custom_proxy],
		['device', device],
		['error_on_404', error_on_404],
		['error_on_redirect', error_on_redirect],
		['js_script', js_script],
		['format', format]
	]) {
		if (v !== undefined && v !== '' && k !== undefined) {
			url_.searchParams.append(k, v)
		}
	}
	encodeParams({ headers }).forEach((v, k) => {
		if (v !== undefined && v !== '') {
			url_.searchParams.append(k, v)
		}
	})
	const response = await fetch(url_, {
		method: 'GET',
		body: undefined
	})
	if (!response.ok) {
		const text = await response.text()
		throw new Error(`${response.status} ${text}`)
	}
	return await response.text()
}

function encodeParams(o: any) {
	function iter(o: any, path: string) {
		if (Array.isArray(o)) {
			o.forEach(function (a) {
				iter(a, path + '[]')
			})
			return
		}
		if (o !== null && typeof o === 'object') {
			Object.keys(o).forEach(function (k) {
				iter(o[k], path + '[' + k + ']')
			})
			return
		}
		data.push(path + '=' + o)
	}
	const data: string[] = []
	Object.keys(o).forEach(function (k) {
		if (o[k] !== undefined) {
			iter(o[k], k)
		}
	})
	return new URLSearchParams(data.join('&'))
}


`1`	`//native`
`2`	`type Webscrapingai = {`
`3`	`apiKey: string`
`4`	`}`
`5`	`/**`
`6`	`* Get an answer to a question about a given web page`
`7`	`* Returns the answer in plain text. Proxies and Chromium JavaScript rendering are used for page retrieval and processing, then the answer is extracted using an LLM model.`
`8`	`*/`
`9`	`export async function main(`
`10`	`auth: Webscrapingai,`
`11`	`url: string \| undefined,`
`12`	`question: string \| undefined,`
`13`	`context_limit: '4000' \| '8000' \| '16000' \| undefined,`
`14`	`response_tokens: string \| undefined,`
`15`	`on_context_limit: 'truncate' \| 'error' \| undefined,`
`16`	`headers: any,`
`17`	`timeout: string \| undefined,`
`18`	`js: string \| undefined,`
`19`	`js_timeout: string \| undefined,`
`20`	`wait_for: string \| undefined,`
`21`	`proxy: 'datacenter' \| 'residential' \| undefined,`
`22`	`country: 'us' \| 'gb' \| 'de' \| 'it' \| 'fr' \| 'ca' \| 'es' \| 'ru' \| 'jp' \| 'kr' \| 'in' \| undefined,`
`23`	`custom_proxy: string \| undefined,`
`24`	`device: 'desktop' \| 'mobile' \| 'tablet' \| undefined,`
`25`	`error_on_404: string \| undefined,`
`26`	`error_on_redirect: string \| undefined,`
`27`	`js_script: string \| undefined,`
`28`	`format: 'json' \| 'text' \| undefined`
`29`	`) {`
`30`	const url_ = new URL(`https://api.webscraping.ai/ai/question`)
`31`
`32`	`url_.searchParams.append('api_key', auth.apiKey)`
`33`
`34`	`for (const [k, v] of [`
`35`	`['url', url],`
`36`	`['question', question],`
`37`	`['context_limit', context_limit],`
`38`	`['response_tokens', response_tokens],`
`39`	`['on_context_limit', on_context_limit],`
`40`	`['timeout', timeout],`
`41`	`['js', js],`
`42`	`['js_timeout', js_timeout],`
`43`	`['wait_for', wait_for],`
`44`	`['proxy', proxy],`
`45`	`['country', country],`
`46`	`['custom_proxy', custom_proxy],`
`47`	`['device', device],`
`48`	`['error_on_404', error_on_404],`
`49`	`['error_on_redirect', error_on_redirect],`
`50`	`['js_script', js_script],`
`51`	`['format', format]`
`52`	`]) {`
`53`	`if (v !== undefined && v !== '' && k !== undefined) {`
`54`	`url_.searchParams.append(k, v)`
`55`	`}`
`56`	`}`
`57`	`encodeParams({ headers }).forEach((v, k) => {`
`58`	`if (v !== undefined && v !== '') {`
`59`	`url_.searchParams.append(k, v)`
`60`	`}`
`61`	`})`
`62`	`const response = await fetch(url_, {`
`63`	`method: 'GET',`
`64`	`body: undefined`
`65`	`})`
`66`	`if (!response.ok) {`
`67`	`const text = await response.text()`
`68`	throw new Error(`${response.status} ${text}`)
`69`	`}`
`70`	`return await response.text()`
`71`	`}`
`72`
`73`	`function encodeParams(o: any) {`
`74`	`function iter(o: any, path: string) {`
`75`	`if (Array.isArray(o)) {`
`76`	`o.forEach(function (a) {`
`77`	`iter(a, path + '[]')`
`78`	`})`
`79`	`return`
`80`	`}`
`81`	`if (o !== null && typeof o === 'object') {`
`82`	`Object.keys(o).forEach(function (k) {`
`83`	`iter(o[k], path + '[' + k + ']')`
`84`	`})`
`85`	`return`
`86`	`}`
`87`	`data.push(path + '=' + o)`
`88`	`}`
`89`	`const data: string[] = []`
`90`	`Object.keys(o).forEach(function (k) {`
`91`	`if (o[k] !== undefined) {`
`92`	`iter(o[k], k)`
`93`	`}`
`94`	`})`
`95`	`return new URLSearchParams(data.join('&'))`
`96`	`}`
`97`