I need assistance with a Python script. When it opens a link in Chrome, the code seems to restart without any error message. Can someone help me

vittorianicolosi1994

Regular Member
Joined
Jan 17, 2024
Messages
279
Reaction score
81
i cannot figure out what happen .
it should scrapes website ,it is python code use selenium undetected chromedriver


I was new to web scraping and I was trying to create a scraper which scarpe data and otput json file.

But the python kept rejecting my connection after open chrome and succesful open the link

It 's not worked. But idk the problem

******
THIS IS THE CODE:
import undetected_chromedriver as uc
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException, TimeoutException
import json
from dataclasses import dataclass
from typing import List, Union
import time
import os

scrape_delay = 20

@dataclass
class Monster:
Name: str
Attribute: str
Type: List[str]
Lvl: str
ATK: str
DEF: str
Arrows: List[str]
Scales: str
PEffect: str
MEffect: str

def to_dict(self):
return {
"Name": self.Name,
"Attribute": self.Attribute,
"Type": self.Type,
"Level": self.Lvl,
"ATK": self.ATK,
"DEF": self.DEF,
"Arrows": self.Arrows,
"Scales": self.Scales,
"PEffect": self.PEffect,
"MEffect": self.MEffect
}

@dataclass
class Backrow:
Name: str
Type: str
Subtype: str
Effect: str

def to_dict(self):
return {
"Name": self.Name,
"Type": self.Type,
"Subtype": self.Subtype,
"Effect": self.Effect
}

def buf_read(prompt: str, to_int: bool = False) -> Union[str, int]:
while True:
user_input = input(prompt).strip()
if to_int:
try:
return int(user_input)
except ValueError:
print("Invalid input. Please enter a valid integer.")
continue
return user_input

def handle_filtering(elements):
sifted = []
for element in elements:
try:
element.find_element(By.CSS_SELECTOR, ".custom")
sifted.append(element)
except NoSuchElementException:
continue
return sifted

def handle_link_arrows(element):
red_arrows = []
directions = [
"Top-Left", "Top-Middle", "Top-Right",
"Middle-Right", "Bottom-Right",
"Bottom-Middle", "Bottom-Left", "Middle-Left"
]
arrows = element.find_elements(By.CSS_SELECTOR, ".red_arrow")
for i, arrow in enumerate(arrows):
if 'display: none' not in arrow.get_attribute('style'):
red_arrows.append(directions)
return red_arrows

def monster_data(element):
m_name = element.find_element(By.CSS_SELECTOR, ".name_txt").text
attri_src = element.find_element(By.CSS_SELECTOR, ".attribute").get_attribute('src')
attri_split = attri_src.split('/')[-1].split('_')[0]
m_type_txt = element.find_element(By.CSS_SELECTOR, ".type_txt").text
m_type = [t.title() for t in m_type_txt.replace(' / ', ' ').split()]

monster_card = element.find_element(By.CSS_SELECTOR, ".card_color").get_attribute('src')
atk_stat = element.find_element(By.CSS_SELECTOR, ".card_atk_txt").text
def_stat = "NaN"
level = "0"
arrows = []

if "link_front" in monster_card:
def_stat = "NaN"
level = element.find_element(By.CSS_SELECTOR, ".link_txt").text
arrows = handle_link_arrows(element)
elif "xyz_front" in monster_card:
level = str(len(element.find_elements(By.CSS_SELECTOR, ".rank")))
else:
level = str(len(element.find_elements(By.CSS_SELECTOR, ".level")))

pend_front = element.find_element(By.CSS_SELECTOR, ".pendulum_front")
pend_style = pend_front.get_attribute('style')
pend_eff = "NaN"
pend_scales = "NaN"

if 'display: block' in pend_style:
pend_eff = element.find_element(By.CSS_SELECTOR, ".card_pendulum_effect_txt").text
pend_scales = element.find_element(By.CSS_SELECTOR, ".scale_left_txt").text

m_effect = element.find_element(By.CSS_SELECTOR, ".effect_txt").text

return Monster(
Name=m_name,
Attribute=attri_split.upper(),
Type=m_type,
Lvl=level,
ATK=atk_stat,
DEF=def_stat,
Arrows=arrows,
Scales=pend_scales,
PEffect=pend_eff,
MEffect=m_effect
)

def backrow_data(element, backrow_type):
backrow_name = element.find_element(By.CSS_SELECTOR, ".name_txt").text
sub_hold = element.find_element(By.CSS_SELECTOR, ".type_icon")
sub_style = sub_hold.get_attribute('style')
sub_type = "Normal"

if "display: block" in sub_style:
sub_src = sub_hold.get_attribute('src')
if "continuous" in sub_src:
sub_type = "Cont."
elif "field" in sub_src:
sub_type = "Field"
elif "equip" in sub_src:
sub_type = "Equip"
elif "quick-play" in sub_src:
sub_type = "Quick-Play"
elif "ritual" in sub_src:
sub_type = "Ritual"
else:
sub_type = "Counter"

backrow_eff = element.find_element(By.CSS_SELECTOR, ".effect_txt").text
return Backrow(
Name=backrow_name,
Type=backrow_type,
Subtype=sub_type,
Effect=backrow_eff
)

def handle_extraction(elements):
monster_arr = []
backrow_arr = []
for el_group in elements:
filtered = handle_filtering(el_group)
for v in filtered:
try:
card_type = v.find_element(By.CSS_SELECTOR, ".attribute").get_attribute('src')
if "spell_attribute" in card_type:
backrow_arr.append(backrow_data(v, "Spell"))
elif "trap_attribute" in card_type:
backrow_arr.append(backrow_data(v, "Trap"))
else:
monster_arr.append(monster_data(v))
except Exception as e:
print(f"Error processing element: {e}")
continue
return monster_arr, backrow_arr

def db_deck_scrape(driver, url):
try:
driver.get(url)
WebDriverWait(driver, scrape_delay).until(
EC.presence_of_element_located((By.CSS_SELECTOR, ".cards"))
)
section = driver.find_element(By.CSS_SELECTOR, ".cards")
master_deck = [
section.find_elements(By.CSS_SELECTOR, ".deck_card"),
section.find_elements(By.CSS_SELECTOR, ".side_card"),
section.find_elements(By.CSS_SELECTOR, ".extra_card")
]

if not any(master_deck):
print("\nERROR: Couldn't scrape the needed data from the provided deck link!")
print("Perhaps the site is slow/down? Try waiting a few minutes or increasing the scrape delay.")
return

mon, bac = handle_extraction(master_deck)
built_string = format_text_data(mon, bac)

file_name = input("\nWhat should the saved file be called?: ").strip()
with open(f"{file_name}.txt", "w") as f:
f.write(built_string)

if input("\nDo you want the JSON file too? (Y/N): ").strip().lower() in ['y', 'yes']:
with open(f"{file_name}.json", "w") as f:
json.dump({
"Monsters": [m.to_dict() for m in mon],
"Backrow": [b.to_dict() for b in bac]
}, f, indent=2)
except Exception as e:
print(f"Scraping failed: {str(e)}")

def format_text_data(monsters, backrows):
output = []
output.append("=== MONSTERS ===")
for mon in monsters:
output.append(f"Name: {mon.Name}")
output.append(f"Attribute: {mon.Attribute}")
output.append(f"Type: {', '.join(mon.Type)}")
output.append(f"Level/Rank/Link: {mon.Lvl}")
output.append(f"ATK: {mon.ATK}")
output.append(f"DEF: {mon.DEF}")
if mon.Arrows:
output.append(f"Link Arrows: {', '.join(mon.Arrows)}")
if mon.Scales != "NaN":
output.append(f"Pendulum Scale: {mon.Scales}")
if mon.PEffect != "NaN":
output.append(f"Pendulum Effect: {mon.PEffect}")
output.append(f"Monster Effect: {mon.MEffect}")
output.append("-" * 40)

output.append("\n=== BACKROW ===")
for bac in backrows:
output.append(f"Name: {bac.Name}")
output.append(f"Type: {bac.Type}")
output.append(f"Subtype: {bac.Subtype}")
output.append(f"Effect: {bac.Effect}")
output.append("-" * 40)

return "\n".join(output)

def show_notes():
notes = """
NOTES:
1. Ensure Chrome is installed and compatible with chromedriver
2. Use valid DuelingBook deck URLs containing /deck?id=
3. Increase scrape delay if encountering timeouts
4. Some card data might require manual verification
"""
print(notes)
input("Press Enter to continue...")

def main():


while True:
os.system('cls' if os.name == 'nt' else 'clear')
print(title)
print(f"[1] Start scraping")

print("[3] Show notes")
print("[4] Exit program")

choice = buf_read("\nEnter your choice: ", to_int=True)

if choice == 1:
url = buf_read("Enter DuelingBook deck URL: ").strip()
if "duelingbook.com/deck?id=" not in url.lower():
print("ERROR: Invalid DuelingBook URL format!")
time.sleep(3)
continue

print(f"Starting scrape with s delay...")
options = uc.ChromeOptions()
driver = uc.Chrome(options=options)
try:
db_deck_scrape(driver, url)
finally:
driver.quit()

elif choice == 2:
new_delay = buf_read(f"Enter new delay (current:): ", to_int=True)
if new_delay < 5:
print("Minimum delay is 5 seconds")
new_delay = 5
scrape_delay = new_delay
print(f"Scrape delay updated to seconds")
time.sleep(2)

elif choice == 3:
show_notes()

elif choice == 4:
print("Exiting program...")
break

else:
print("Invalid choice. Please select 1-4")
time.sleep(2)

if __name__ == "__main__":
main()
******
THIS IS THE WEBSITE:https://shorturl.at/wUbfJ
******
this is the input url: https://www.duelingbook.com/deck?id=6181911
 
It might be the site's anti-scraping measures kicking in; try adding some randomized delays between requests to see if that helps.
 
Back
Top