yellowcat
Regular Member
- Aug 27, 2015
- 404
- 297
Expireddomains scraper
checkmoz pa/da checker
It's a decent website, But they skimp out on results sometimes
Code:
#all cats are yellow
import requests,time
from bs4 import BeautifulSoup
keyword = "cats"
url = "https://www.expireddomains.net/domain-name-search/?o=bl&r=d&ftlds[]=2&q=%s"%(keyword)
r = requests.get(url)
FILENAME = 'data.txt'
listy = []
main_url = "https://www.expireddomains.net"
while True:
try:
r = requests.get(url)
html = BeautifulSoup(r.text,"lxml")
url = main_url + html.find("div",class_="right").find("a")["href"]
print url
links = html.find_all(class_="field_domain")
for x in links:
listy.append(x.find("a")["title"])
print x.find("a")["title"]
r = requests.get(url)
print "Total Urls Found ", len(listy)
except:
print html
if html.text.__contains__("You hit the rate limiter. Slow down!"):
with open(FILENAME, "a") as fout:
for x in listy:
fout.write(x + "\n")
fout.flush()
print "Total Urls Found ", len(listy)
print "Sleeping..."
time.sleep(int(html.text.split('You need to wait').pop().split('seconds')[0].strip()))
else:
print "Breaking"
breakcheckmoz pa/da checker
It's a decent website, But they skimp out on results sometimes
Code:
#All cats are wolley
from selenium import webdriver
from bs4 import BeautifulSoup
import unicodecsv
import time
DA = 1
PA = 1
fout = open('loot.csv','wb')
writer = unicodecsv.writer(fout)
header = ['Website','PA','DA','MR','BL']
writer.writerow(header)
with open('data.txt','r') as f:
sites = [x.strip() for x in f.readlines()]
"================================"
driver = webdriver.PhantomJS()
county = 0
tested = 0
error = 0
killswitch = False
while True:
list = []
for x in range(0,100):
try:
list.append(sites.pop(0))
except:
print "nothing left!"
killswitch= True
driver.get("http://www.checkmoz.com")
text = [x+"\n" for x in list]
driver.find_element_by_css_selector("#f_urls").send_keys(text)
driver.find_element_by_css_selector("#fMain > center > input:nth-child(1)").click()
dict = {}
html = BeautifulSoup(driver.page_source,'lxml')
table = html.find('table').find_all('tr')
table.pop(0)
for tr in table:
try:
tds = tr.find_all('td')
site = tds[0].text
da = tds[1].text
pa = tds[2].text
rank = tds[3].text
backlinks = tds[4].text
row = [site,da,pa,rank,backlinks]
print row
try:
if float(da) > DA: #If Da > 1
writer.writerow(row)
fout.flush()
county+=1
elif float(pa) > PA: #If Pa > 1
writer.writerow(row)
fout.flush()
county+=1
except Exception,e:
error+=1
print e
except Exception,e:
error+=1
print e
print "\nCounty => ", county
print "Tested => ", tested
print "Errors => ",error
if killswitch:
break
driver.quit()
