FantasticalMystery
Newbie
- Mar 19, 2024
- 17
- 7
Hello! Im new to scraping and need a bit of help. Ive got my code written out but its not necessarily working. Basically, its a scraper for goodreads and its supposed to scrape information from a specific author once I enter the author name in the terminal but I just cant seem to figure out whats not working and why its just giving me a "no books found" error. Any help would be greatly appreciated, Thanks!
Python:
from datetime import datetime
import requests
from bs4 import BeautifulSoup
import json
def fetch_page(url):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
}
response = requests.get(url, headers=headers)
if response.status_code == 200:
return BeautifulSoup(response.content, 'html.parser')
else:
print(f"Failed to fetch page. Status code: {response.status_code}")
return None
def get_timestamp():
return datetime.now().strftime('%Y-%m-%d %H:%M:%S')
def scrape_search_results(search_url):
soup = fetch_page(search_url)
if soup is None:
print("Failed to fetch the page or parse the content.")
return []
titles = soup.find_all('a', class_='bookTitle')
authors = soup.find_all('a', class_='authorName')
avg_ratings = soup.find_all('span', class_='minirating')
if not titles or not authors or not avg_ratings:
print("Couldn't find the necessary data on the page.")
return []
books = []
for title, author, rating in zip(titles, authors, avg_ratings):
book = {
"title": title.text.strip(),
"author": author.text.strip(),
"avg_rating": rating.text.strip().split(' — ')[0].replace('avg rating', '').strip(),
"numb_rating": rating.text.strip().split(' — ')[1].replace('ratings', '').strip()
}
books.append(book)
return books
search_url = f'https://www.goodreads.com/search?q=[def search_kw]'
def save_to_json(data, filename='books.json'):
with open(filename, 'w', encoding='utf-8') as f:
json.dump(data, f, ensure_ascii=False, indent=4)
print(f"Data saved to {filename}")
books = scrape_search_results(search_url)
if books:
for book in books:
print(book)
else:
print("No books were found.")