web scraping example

from typing import List, Dict

import requests

from bs4 import BeautifulSoup

def main(url: str) -> List[Dict[str, str]]:

    # Send a GET request to the URL

    headers = {

        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",

        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",

        "Accept-Language": "en-US,en;q=0.5",

        "Accept-Encoding": "gzip, deflate, br",

        "Connection": "keep-alive",

        "Upgrade-Insecure-Requests": "1",

        "Cache-Control": "max-age=0",

        "TE": "Trailers",

        "DNT": "1",  # Do Not Track

        "Sec-Fetch-Dest": "document",

        "Sec-Fetch-Mode": "navigate",

        "Sec-Fetch-Site": "none",

        "Sec-Fetch-User": "?1",

    }

    response = requests.get(url, headers=headers)

    # Parse the HTML content using BeautifulSoup

    soup = BeautifulSoup(response.content, "html.parser")

    # Find the section with class "quickfinder"

    quickfinder_section = soup.find(class_="quickfinder")

    # If the section is found, get all 'a' tags within it

    if quickfinder_section:

        a_tags = quickfinder_section.find_all("a", recursive=True)

        results = [

            {"text": a.get_text(strip=True), "href": a.get("href")}

            for a in a_tags

            if a.get_text(strip=True)

        ]

    else:

        results = []

    return results