import mechanize
from bs4 import BeautifulSoup
def remove_duplicates(values):
output = []
seen = set()
for value in values:
# If value has not been encountered yet,
# ... add it to both list and set.
if value not in seen:
output.append(value)
seen.add(value)
return output
def EasyFind(text,begin,end):
b = text.find(begin) + len(begin)
e = b + text[b:].find(end)
return text[b:e]
def scrapeInfo(url):
br = mechanize.Browser()
response = br.open(url)
html = response.read()
if "ApiResponse = " not in html:
return
phones = []
contacts = []
emails = []
website = ""
name = EasyFind(html, '{"name":"', '"')
if 'dealerWebsiteLink":"' in html:
website = EasyFind(html,'dealerWebsiteLink":"','"').replace("?utm_source=cars.com&utm_medium=referral","")
if 'newCarEmail":"' in html:
newEmail = EasyFind(html, 'newCarEmail":"','"')
if not "leads" in newEmail:
emails.append(newEmail)
if 'usedCarEmail":"' in html:
usedEmail = EasyFind(html,'usedCarEmail":"','"')
if not "leads" in usedEmail:
emails.append(usedEmail)
if 'usedPhoneNumber":"' in html:
usedPhone = EasyFind(html, 'usedPhoneNumber":"', '"')
if len(usedPhone) == 12:
phones.append(usedPhone)
if 'servicePhoneNumber":"' in html:
serPhone = EasyFind(html,'servicePhoneNumber":"','"')
if len(serPhone) == 12:
phones.append(serPhone)
if 'newPhoneNumber":"' in html:
newPhone = EasyFind(html,'newPhoneNumber":"','"')
if len(newPhone) == 12:
phones.append(newPhone)
if 'usedCarContactName":"' in html:
usedContact = EasyFind(html,'usedCarContactName":"','"')
contacts.append(usedContact)
if 'newCarContactName":"' in html:
newContact = EasyFind(html,'newCarContactName":"','"')
contacts.append(newContact)
city = EasyFind(html,'city":"','"')
state = EasyFind(html,'state":"','"')
emails = remove_duplicates(emails)
contacts = remove_duplicates(contacts)
phones = remove_duplicates(phones)
if phones and emails and contacts:
return "Dealership: %s|Website: %s|%s, %s|%s|%s|%s"%(name,website,city,state,str(phones),str(emails),str(contacts))
def scrapePages(num):
for i in range(0,num):
url = "
http://www.cars.com/dealers/search/repair/all/20001/?start=%s&rpcShop=false&radius=100000"%(51*i)
br = mechanize.Browser()
response = br.open(url)
html = response.read()
soup = BeautifulSoup(html)
dealers = soup.find_all("a", {"name":"&lid=dealer-name"})
for dealer in dealers:
d = scrapeInfo('
http://www.cars.com'+dealer['href'])
if not d:
print "No data found on "+'
http://www.cars.com'+dealer['href']
continue
print d
