Please need help extract HTML table for each keyword searched and save them in a CSV file.
The code I used is:
what I need to achieve:
The code I used is:
Code:
import requests
import random
import csv
import time
import numpy
from bs4 import BeautifulSoup
from time import sleep
# Read the keywords from a file
with open("keywords.txt", "r") as file:
keywords = file.read().splitlines()
# Define the User-Agent header
user_agent_list = [
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:77.0) Gecko/20100101 Firefox/77.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)',
'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)',
'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)'
]
for _ in user_agent_list:
#Pick a random user agent
user_agent = random.choice(user_agent_list)
#Set the headers
headers = {'User-Agent': user_agent}
# Create a new CSV file and write the headers
with open("results.csv", "w", newline="") as file:
writer = csv.writer(file)
writer.writerow(["Keyword", "Total Results"])
# Perform the search for each keyword and write the total number of results to the CSV file
for keyword in keywords:
response = requests.get(f"https://ae.godaddy.com/domain-value-appraisal/appraisal/?domainToCheck={keyword}", headers=headers)
soup = BeautifulSoup(response.content, "html.parser")
table = soup.find("table", {"class": "comparable-domains-table"})
if table:
total_results = table.get_text()
writer.writerow([keyword, total_results])
print(f"Keyword: {keyword}, Total Results: {total_results}")
else:
writer.writerow([keyword, "Not found"])
print(f"Keyword: {keyword}, Total Results: Not found")
what I need to achieve:
- get the keyword from keywords.txt
- extracting the table class="comparable-domains-table" who has three tr and two td
- Save data in CSV file