yellowcat
Regular Member
- Aug 27, 2015
- 404
- 297
Made a quick script today to scrape expireddomains.net (Hacked it together in 10 mins don't judge)
Simply just go to expireddomains enter in your search query and click search.
Replace "URL_HERE" with your url
Here's a sample url
https://www.expireddomains.net/domain-name-search/?o=bl&r=d&ftlds[]=2&q=cats
Simply just go to expireddomains enter in your search query and click search.
Replace "URL_HERE" with your url
Here's a sample url
https://www.expireddomains.net/domain-name-search/?o=bl&r=d&ftlds[]=2&q=cats
Code:
#all cats are yellow
import requests,time
from bs4 import BeautifulSoup
url = "URL_HERE"
r = requests.get(url)
listy = []
main_url = "https://www.expireddomains.net"
while True:
try:
r = requests.get(url)
html = BeautifulSoup(r.text,"lxml")
url = main_url + html.find("div",class_="right").find("a")["href"]
print url
links = html.find_all(class_="field_domain")
for x in links:
listy.append(x.find("a")["title"])
print x.find("a")["title"]
r = requests.get(url)
print "Total Urls Found ", len(listy)
except:
print html
if html.text.__contains__("You hit the rate limiter. Slow down!"):
File = open("sites.txt", "w")
for x in listy:
File.write(x + "\n")
File.flush()
print "Total Urls Found ", len(listy)
print "Sleeping..."
time.sleep(5)
else:
print "Breaking"
break
File = open("sites.txt","w")
for x in listy:
File.write(x + "\n")
File.close()