Edits history of script submission #8891 for ' web scraping example (github)'

• python3

  Edit in Windmill

  Embed

  from typing import List, Dict
  import requests
  from bs4 import BeautifulSoup
  
  
  def main(url: str) -> List[Dict[str, str]]:
      # Send a GET request to the URL
      headers = {
          "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
          "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
          "Accept-Language": "en-US,en;q=0.5",
          "Accept-Encoding": "gzip, deflate, br",
          "Connection": "keep-alive",
          "Upgrade-Insecure-Requests": "1",
          "Cache-Control": "max-age=0",
          "TE": "Trailers",
          "DNT": "1",  # Do Not Track
          "Sec-Fetch-Dest": "document",
          "Sec-Fetch-Mode": "navigate",
          "Sec-Fetch-Site": "none",
          "Sec-Fetch-User": "?1",
      }
      response = requests.get(url, headers=headers)
  
      # Parse the HTML content using BeautifulSoup
      soup = BeautifulSoup(response.content, "html.parser")
  
      # Find the section with class "quickfinder"
      quickfinder_section = soup.find(class_="quickfinder")
  
      # If the section is found, get all 'a' tags within it
      if quickfinder_section:
          a_tags = quickfinder_section.find_all("a", recursive=True)
          results = [
              {"text": a.get_text(strip=True), "href": a.get("href")}
              for a in a_tags
              if a.get_text(strip=True)
          ]
      else:
          results = []
  
      return results
  

  Submitted by ksa.mo7md22 658 days ago