Need help with some code/ Newbie

Joined
Mar 19, 2024
Messages
17
Reaction score
7
Hello! Im new to scraping and need a bit of help. Ive got my code written out but its not necessarily working. Basically, its a scraper for goodreads and its supposed to scrape information from a specific author once I enter the author name in the terminal but I just cant seem to figure out whats not working and why its just giving me a "no books found" error. Any help would be greatly appreciated, Thanks!

Python:
from datetime import datetime

import requests
from bs4 import BeautifulSoup
import json


def fetch_page(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
    }


    response = requests.get(url, headers=headers)


    if response.status_code == 200:
        return BeautifulSoup(response.content, 'html.parser')
    else:
        print(f"Failed to fetch page. Status code: {response.status_code}")
        return None

def get_timestamp():
    return datetime.now().strftime('%Y-%m-%d %H:%M:%S')

def scrape_search_results(search_url):
    soup = fetch_page(search_url)

    if soup is None:
        print("Failed to fetch the page or parse the content.")
        return []


    titles = soup.find_all('a', class_='bookTitle')
    authors = soup.find_all('a', class_='authorName')
    avg_ratings = soup.find_all('span', class_='minirating')

    if not titles or not authors or not avg_ratings:
        print("Couldn't find the necessary data on the page.")
        return []

    books = []
    for title, author, rating in zip(titles, authors, avg_ratings):
        book = {
            "title": title.text.strip(),
            "author": author.text.strip(),
            "avg_rating": rating.text.strip().split(' — ')[0].replace('avg rating', '').strip(),
            "numb_rating": rating.text.strip().split(' — ')[1].replace('ratings', '').strip()
        }
        books.append(book)

    return books


search_url = f'https://www.goodreads.com/search?q=[def search_kw]'

def save_to_json(data, filename='books.json'):
    with open(filename, 'w', encoding='utf-8') as f:
        json.dump(data, f, ensure_ascii=False, indent=4)
    print(f"Data saved to {filename}")

books = scrape_search_results(search_url)

if books:
    for book in books:
        print(book)
else:
    print("No books were found.")
 
Do you mind share screenshots of the error?
 
Back
Top