Need help extract table from website

alifihri

Newbie
Joined
Apr 9, 2023
Messages
7
Reaction score
1
Please need help extract HTML table for each keyword searched and save them in a CSV file.
The code I used is:

Code:
import requests
import random
import csv
import time
import numpy
from bs4 import BeautifulSoup
from time import sleep


# Read the keywords from a file
with open("keywords.txt", "r") as file:
    keywords = file.read().splitlines()

# Define the User-Agent header
user_agent_list = [
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15',
  'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0',
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36',
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:77.0) Gecko/20100101 Firefox/77.0',
  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36',
  'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36',
  'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36',
  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
  'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
  'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)',
  'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)',
  'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)'
]
for _ in user_agent_list:
    
  #Pick a random user agent
  user_agent = random.choice(user_agent_list)

  #Set the headers
  headers = {'User-Agent': user_agent}


# Create a new CSV file and write the headers
with open("results.csv", "w", newline="") as file:
    writer = csv.writer(file)
    writer.writerow(["Keyword", "Total Results"])

    # Perform the search for each keyword and write the total number of results to the CSV file
    for keyword in keywords:
        response = requests.get(f"https://ae.godaddy.com/domain-value-appraisal/appraisal/?domainToCheck={keyword}", headers=headers)
        soup = BeautifulSoup(response.content, "html.parser")
        table = soup.find("table", {"class": "comparable-domains-table"})
        if table:
            total_results = table.get_text()
            writer.writerow([keyword, total_results])
            print(f"Keyword: {keyword}, Total Results: {total_results}")
        else:
            writer.writerow([keyword, "Not found"])
            print(f"Keyword: {keyword}, Total Results: Not found")

what I need to achieve:
  1. get the keyword from keywords.txt
  2. extracting the table class="comparable-domains-table" who has three tr and two td
  3. Save data in CSV file
 
Back
Top