vittorianicolosi1994
Regular Member
- Jan 17, 2024
- 279
- 81
i cannot figure out what happen .
it should scrapes website ,it is python code use selenium undetected chromedriver
I was new to web scraping and I was trying to create a scraper which scarpe data and otput json file.
But the python kept rejecting my connection after open chrome and succesful open the link
It 's not worked. But idk the problem
******
THIS IS THE CODE:
import undetected_chromedriver as uc
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException, TimeoutException
import json
from dataclasses import dataclass
from typing import List, Union
import time
import os
scrape_delay = 20
@dataclass
class Monster:
Name: str
Attribute: str
Type: List[str]
Lvl: str
ATK: str
DEF: str
Arrows: List[str]
Scales: str
PEffect: str
MEffect: str
def to_dict(self):
return {
"Name": self.Name,
"Attribute": self.Attribute,
"Type": self.Type,
"Level": self.Lvl,
"ATK": self.ATK,
"DEF": self.DEF,
"Arrows": self.Arrows,
"Scales": self.Scales,
"PEffect": self.PEffect,
"MEffect": self.MEffect
}
@dataclass
class Backrow:
Name: str
Type: str
Subtype: str
Effect: str
def to_dict(self):
return {
"Name": self.Name,
"Type": self.Type,
"Subtype": self.Subtype,
"Effect": self.Effect
}
def buf_read(prompt: str, to_int: bool = False) -> Union[str, int]:
while True:
user_input = input(prompt).strip()
if to_int:
try:
return int(user_input)
except ValueError:
print("Invalid input. Please enter a valid integer.")
continue
return user_input
def handle_filtering(elements):
sifted = []
for element in elements:
try:
element.find_element(By.CSS_SELECTOR, ".custom")
sifted.append(element)
except NoSuchElementException:
continue
return sifted
def handle_link_arrows(element):
red_arrows = []
directions = [
"Top-Left", "Top-Middle", "Top-Right",
"Middle-Right", "Bottom-Right",
"Bottom-Middle", "Bottom-Left", "Middle-Left"
]
arrows = element.find_elements(By.CSS_SELECTOR, ".red_arrow")
for i, arrow in enumerate(arrows):
if 'display: none' not in arrow.get_attribute('style'):
red_arrows.append(directions)
return red_arrows
def monster_data(element):
m_name = element.find_element(By.CSS_SELECTOR, ".name_txt").text
attri_src = element.find_element(By.CSS_SELECTOR, ".attribute").get_attribute('src')
attri_split = attri_src.split('/')[-1].split('_')[0]
m_type_txt = element.find_element(By.CSS_SELECTOR, ".type_txt").text
m_type = [t.title() for t in m_type_txt.replace(' / ', ' ').split()]
monster_card = element.find_element(By.CSS_SELECTOR, ".card_color").get_attribute('src')
atk_stat = element.find_element(By.CSS_SELECTOR, ".card_atk_txt").text
def_stat = "NaN"
level = "0"
arrows = []
if "link_front" in monster_card:
def_stat = "NaN"
level = element.find_element(By.CSS_SELECTOR, ".link_txt").text
arrows = handle_link_arrows(element)
elif "xyz_front" in monster_card:
level = str(len(element.find_elements(By.CSS_SELECTOR, ".rank")))
else:
level = str(len(element.find_elements(By.CSS_SELECTOR, ".level")))
pend_front = element.find_element(By.CSS_SELECTOR, ".pendulum_front")
pend_style = pend_front.get_attribute('style')
pend_eff = "NaN"
pend_scales = "NaN"
if 'display: block' in pend_style:
pend_eff = element.find_element(By.CSS_SELECTOR, ".card_pendulum_effect_txt").text
pend_scales = element.find_element(By.CSS_SELECTOR, ".scale_left_txt").text
m_effect = element.find_element(By.CSS_SELECTOR, ".effect_txt").text
return Monster(
Name=m_name,
Attribute=attri_split.upper(),
Type=m_type,
Lvl=level,
ATK=atk_stat,
DEF=def_stat,
Arrows=arrows,
Scales=pend_scales,
PEffect=pend_eff,
MEffect=m_effect
)
def backrow_data(element, backrow_type):
backrow_name = element.find_element(By.CSS_SELECTOR, ".name_txt").text
sub_hold = element.find_element(By.CSS_SELECTOR, ".type_icon")
sub_style = sub_hold.get_attribute('style')
sub_type = "Normal"
if "display: block" in sub_style:
sub_src = sub_hold.get_attribute('src')
if "continuous" in sub_src:
sub_type = "Cont."
elif "field" in sub_src:
sub_type = "Field"
elif "equip" in sub_src:
sub_type = "Equip"
elif "quick-play" in sub_src:
sub_type = "Quick-Play"
elif "ritual" in sub_src:
sub_type = "Ritual"
else:
sub_type = "Counter"
backrow_eff = element.find_element(By.CSS_SELECTOR, ".effect_txt").text
return Backrow(
Name=backrow_name,
Type=backrow_type,
Subtype=sub_type,
Effect=backrow_eff
)
def handle_extraction(elements):
monster_arr = []
backrow_arr = []
for el_group in elements:
filtered = handle_filtering(el_group)
for v in filtered:
try:
card_type = v.find_element(By.CSS_SELECTOR, ".attribute").get_attribute('src')
if "spell_attribute" in card_type:
backrow_arr.append(backrow_data(v, "Spell"))
elif "trap_attribute" in card_type:
backrow_arr.append(backrow_data(v, "Trap"))
else:
monster_arr.append(monster_data(v))
except Exception as e:
print(f"Error processing element: {e}")
continue
return monster_arr, backrow_arr
def db_deck_scrape(driver, url):
try:
driver.get(url)
WebDriverWait(driver, scrape_delay).until(
EC.presence_of_element_located((By.CSS_SELECTOR, ".cards"))
)
section = driver.find_element(By.CSS_SELECTOR, ".cards")
master_deck = [
section.find_elements(By.CSS_SELECTOR, ".deck_card"),
section.find_elements(By.CSS_SELECTOR, ".side_card"),
section.find_elements(By.CSS_SELECTOR, ".extra_card")
]
if not any(master_deck):
print("\nERROR: Couldn't scrape the needed data from the provided deck link!")
print("Perhaps the site is slow/down? Try waiting a few minutes or increasing the scrape delay.")
return
mon, bac = handle_extraction(master_deck)
built_string = format_text_data(mon, bac)
file_name = input("\nWhat should the saved file be called?: ").strip()
with open(f"{file_name}.txt", "w") as f:
f.write(built_string)
if input("\nDo you want the JSON file too? (Y/N): ").strip().lower() in ['y', 'yes']:
with open(f"{file_name}.json", "w") as f:
json.dump({
"Monsters": [m.to_dict() for m in mon],
"Backrow": [b.to_dict() for b in bac]
}, f, indent=2)
except Exception as e:
print(f"Scraping failed: {str(e)}")
def format_text_data(monsters, backrows):
output = []
output.append("=== MONSTERS ===")
for mon in monsters:
output.append(f"Name: {mon.Name}")
output.append(f"Attribute: {mon.Attribute}")
output.append(f"Type: {', '.join(mon.Type)}")
output.append(f"Level/Rank/Link: {mon.Lvl}")
output.append(f"ATK: {mon.ATK}")
output.append(f"DEF: {mon.DEF}")
if mon.Arrows:
output.append(f"Link Arrows: {', '.join(mon.Arrows)}")
if mon.Scales != "NaN":
output.append(f"Pendulum Scale: {mon.Scales}")
if mon.PEffect != "NaN":
output.append(f"Pendulum Effect: {mon.PEffect}")
output.append(f"Monster Effect: {mon.MEffect}")
output.append("-" * 40)
output.append("\n=== BACKROW ===")
for bac in backrows:
output.append(f"Name: {bac.Name}")
output.append(f"Type: {bac.Type}")
output.append(f"Subtype: {bac.Subtype}")
output.append(f"Effect: {bac.Effect}")
output.append("-" * 40)
return "\n".join(output)
def show_notes():
notes = """
NOTES:
1. Ensure Chrome is installed and compatible with chromedriver
2. Use valid DuelingBook deck URLs containing /deck?id=
3. Increase scrape delay if encountering timeouts
4. Some card data might require manual verification
"""
print(notes)
input("Press Enter to continue...")
def main():
while True:
os.system('cls' if os.name == 'nt' else 'clear')
print(title)
print(f"[1] Start scraping")
print("[3] Show notes")
print("[4] Exit program")
choice = buf_read("\nEnter your choice: ", to_int=True)
if choice == 1:
url = buf_read("Enter DuelingBook deck URL: ").strip()
if "duelingbook.com/deck?id=" not in url.lower():
print("ERROR: Invalid DuelingBook URL format!")
time.sleep(3)
continue
print(f"Starting scrape with s delay...")
options = uc.ChromeOptions()
driver = uc.Chrome(options=options)
try:
db_deck_scrape(driver, url)
finally:
driver.quit()
elif choice == 2:
new_delay = buf_read(f"Enter new delay (current
: ", to_int=True)
if new_delay < 5:
print("Minimum delay is 5 seconds")
new_delay = 5
scrape_delay = new_delay
print(f"Scrape delay updated to seconds")
time.sleep(2)
elif choice == 3:
show_notes()
elif choice == 4:
print("Exiting program...")
break
else:
print("Invalid choice. Please select 1-4")
time.sleep(2)
if __name__ == "__main__":
main()
******
THIS IS THE WEBSITE:https://shorturl.at/wUbfJ
******
this is the input url: https://www.duelingbook.com/deck?id=6181911
it should scrapes website ,it is python code use selenium undetected chromedriver
I was new to web scraping and I was trying to create a scraper which scarpe data and otput json file.
But the python kept rejecting my connection after open chrome and succesful open the link
It 's not worked. But idk the problem
******
THIS IS THE CODE:
import undetected_chromedriver as uc
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException, TimeoutException
import json
from dataclasses import dataclass
from typing import List, Union
import time
import os
scrape_delay = 20
@dataclass
class Monster:
Name: str
Attribute: str
Type: List[str]
Lvl: str
ATK: str
DEF: str
Arrows: List[str]
Scales: str
PEffect: str
MEffect: str
def to_dict(self):
return {
"Name": self.Name,
"Attribute": self.Attribute,
"Type": self.Type,
"Level": self.Lvl,
"ATK": self.ATK,
"DEF": self.DEF,
"Arrows": self.Arrows,
"Scales": self.Scales,
"PEffect": self.PEffect,
"MEffect": self.MEffect
}
@dataclass
class Backrow:
Name: str
Type: str
Subtype: str
Effect: str
def to_dict(self):
return {
"Name": self.Name,
"Type": self.Type,
"Subtype": self.Subtype,
"Effect": self.Effect
}
def buf_read(prompt: str, to_int: bool = False) -> Union[str, int]:
while True:
user_input = input(prompt).strip()
if to_int:
try:
return int(user_input)
except ValueError:
print("Invalid input. Please enter a valid integer.")
continue
return user_input
def handle_filtering(elements):
sifted = []
for element in elements:
try:
element.find_element(By.CSS_SELECTOR, ".custom")
sifted.append(element)
except NoSuchElementException:
continue
return sifted
def handle_link_arrows(element):
red_arrows = []
directions = [
"Top-Left", "Top-Middle", "Top-Right",
"Middle-Right", "Bottom-Right",
"Bottom-Middle", "Bottom-Left", "Middle-Left"
]
arrows = element.find_elements(By.CSS_SELECTOR, ".red_arrow")
for i, arrow in enumerate(arrows):
if 'display: none' not in arrow.get_attribute('style'):
red_arrows.append(directions)
return red_arrows
def monster_data(element):
m_name = element.find_element(By.CSS_SELECTOR, ".name_txt").text
attri_src = element.find_element(By.CSS_SELECTOR, ".attribute").get_attribute('src')
attri_split = attri_src.split('/')[-1].split('_')[0]
m_type_txt = element.find_element(By.CSS_SELECTOR, ".type_txt").text
m_type = [t.title() for t in m_type_txt.replace(' / ', ' ').split()]
monster_card = element.find_element(By.CSS_SELECTOR, ".card_color").get_attribute('src')
atk_stat = element.find_element(By.CSS_SELECTOR, ".card_atk_txt").text
def_stat = "NaN"
level = "0"
arrows = []
if "link_front" in monster_card:
def_stat = "NaN"
level = element.find_element(By.CSS_SELECTOR, ".link_txt").text
arrows = handle_link_arrows(element)
elif "xyz_front" in monster_card:
level = str(len(element.find_elements(By.CSS_SELECTOR, ".rank")))
else:
level = str(len(element.find_elements(By.CSS_SELECTOR, ".level")))
pend_front = element.find_element(By.CSS_SELECTOR, ".pendulum_front")
pend_style = pend_front.get_attribute('style')
pend_eff = "NaN"
pend_scales = "NaN"
if 'display: block' in pend_style:
pend_eff = element.find_element(By.CSS_SELECTOR, ".card_pendulum_effect_txt").text
pend_scales = element.find_element(By.CSS_SELECTOR, ".scale_left_txt").text
m_effect = element.find_element(By.CSS_SELECTOR, ".effect_txt").text
return Monster(
Name=m_name,
Attribute=attri_split.upper(),
Type=m_type,
Lvl=level,
ATK=atk_stat,
DEF=def_stat,
Arrows=arrows,
Scales=pend_scales,
PEffect=pend_eff,
MEffect=m_effect
)
def backrow_data(element, backrow_type):
backrow_name = element.find_element(By.CSS_SELECTOR, ".name_txt").text
sub_hold = element.find_element(By.CSS_SELECTOR, ".type_icon")
sub_style = sub_hold.get_attribute('style')
sub_type = "Normal"
if "display: block" in sub_style:
sub_src = sub_hold.get_attribute('src')
if "continuous" in sub_src:
sub_type = "Cont."
elif "field" in sub_src:
sub_type = "Field"
elif "equip" in sub_src:
sub_type = "Equip"
elif "quick-play" in sub_src:
sub_type = "Quick-Play"
elif "ritual" in sub_src:
sub_type = "Ritual"
else:
sub_type = "Counter"
backrow_eff = element.find_element(By.CSS_SELECTOR, ".effect_txt").text
return Backrow(
Name=backrow_name,
Type=backrow_type,
Subtype=sub_type,
Effect=backrow_eff
)
def handle_extraction(elements):
monster_arr = []
backrow_arr = []
for el_group in elements:
filtered = handle_filtering(el_group)
for v in filtered:
try:
card_type = v.find_element(By.CSS_SELECTOR, ".attribute").get_attribute('src')
if "spell_attribute" in card_type:
backrow_arr.append(backrow_data(v, "Spell"))
elif "trap_attribute" in card_type:
backrow_arr.append(backrow_data(v, "Trap"))
else:
monster_arr.append(monster_data(v))
except Exception as e:
print(f"Error processing element: {e}")
continue
return monster_arr, backrow_arr
def db_deck_scrape(driver, url):
try:
driver.get(url)
WebDriverWait(driver, scrape_delay).until(
EC.presence_of_element_located((By.CSS_SELECTOR, ".cards"))
)
section = driver.find_element(By.CSS_SELECTOR, ".cards")
master_deck = [
section.find_elements(By.CSS_SELECTOR, ".deck_card"),
section.find_elements(By.CSS_SELECTOR, ".side_card"),
section.find_elements(By.CSS_SELECTOR, ".extra_card")
]
if not any(master_deck):
print("\nERROR: Couldn't scrape the needed data from the provided deck link!")
print("Perhaps the site is slow/down? Try waiting a few minutes or increasing the scrape delay.")
return
mon, bac = handle_extraction(master_deck)
built_string = format_text_data(mon, bac)
file_name = input("\nWhat should the saved file be called?: ").strip()
with open(f"{file_name}.txt", "w") as f:
f.write(built_string)
if input("\nDo you want the JSON file too? (Y/N): ").strip().lower() in ['y', 'yes']:
with open(f"{file_name}.json", "w") as f:
json.dump({
"Monsters": [m.to_dict() for m in mon],
"Backrow": [b.to_dict() for b in bac]
}, f, indent=2)
except Exception as e:
print(f"Scraping failed: {str(e)}")
def format_text_data(monsters, backrows):
output = []
output.append("=== MONSTERS ===")
for mon in monsters:
output.append(f"Name: {mon.Name}")
output.append(f"Attribute: {mon.Attribute}")
output.append(f"Type: {', '.join(mon.Type)}")
output.append(f"Level/Rank/Link: {mon.Lvl}")
output.append(f"ATK: {mon.ATK}")
output.append(f"DEF: {mon.DEF}")
if mon.Arrows:
output.append(f"Link Arrows: {', '.join(mon.Arrows)}")
if mon.Scales != "NaN":
output.append(f"Pendulum Scale: {mon.Scales}")
if mon.PEffect != "NaN":
output.append(f"Pendulum Effect: {mon.PEffect}")
output.append(f"Monster Effect: {mon.MEffect}")
output.append("-" * 40)
output.append("\n=== BACKROW ===")
for bac in backrows:
output.append(f"Name: {bac.Name}")
output.append(f"Type: {bac.Type}")
output.append(f"Subtype: {bac.Subtype}")
output.append(f"Effect: {bac.Effect}")
output.append("-" * 40)
return "\n".join(output)
def show_notes():
notes = """
NOTES:
1. Ensure Chrome is installed and compatible with chromedriver
2. Use valid DuelingBook deck URLs containing /deck?id=
3. Increase scrape delay if encountering timeouts
4. Some card data might require manual verification
"""
print(notes)
input("Press Enter to continue...")
def main():
while True:
os.system('cls' if os.name == 'nt' else 'clear')
print(title)
print(f"[1] Start scraping")
print("[3] Show notes")
print("[4] Exit program")
choice = buf_read("\nEnter your choice: ", to_int=True)
if choice == 1:
url = buf_read("Enter DuelingBook deck URL: ").strip()
if "duelingbook.com/deck?id=" not in url.lower():
print("ERROR: Invalid DuelingBook URL format!")
time.sleep(3)
continue
print(f"Starting scrape with s delay...")
options = uc.ChromeOptions()
driver = uc.Chrome(options=options)
try:
db_deck_scrape(driver, url)
finally:
driver.quit()
elif choice == 2:
new_delay = buf_read(f"Enter new delay (current
if new_delay < 5:
print("Minimum delay is 5 seconds")
new_delay = 5
scrape_delay = new_delay
print(f"Scrape delay updated to seconds")
time.sleep(2)
elif choice == 3:
show_notes()
elif choice == 4:
print("Exiting program...")
break
else:
print("Invalid choice. Please select 1-4")
time.sleep(2)
if __name__ == "__main__":
main()
******
THIS IS THE WEBSITE:https://shorturl.at/wUbfJ
******
this is the input url: https://www.duelingbook.com/deck?id=6181911