HaryanviJakkat
Newbie
- May 19, 2019
- 10
- 3
As you guys asked here we go with my new advance proxy scraping script,if you have any questions just feel free to ask .
Code:
from bs4 import BeautifulSoup
import cfscrape
from pathlib import Path
import os
import time
repeat = 2 # scrape 2 times
time_interval = 10 # seconds between each repeat
while True:
if repeat >= 0:
repeat -= 1
time.sleep(time_interval)
url = "https://hidemyna.me/en/proxy-list/"
scraper = cfscrape.create_scraper()
html = scraper.get(url).content
soup = BeautifulSoup(html, 'lxml')
table = soup.find("tbody")
for listt in table:
for proxy in listt.findAll("td")[0:1]:
for port in listt.findAll("td")[1:2]:
# print(proxy, port)
proxylist = f"{proxy.text}:{port.text}"
with open('proxies.txt', 'a+') as f:
f.writelines(proxylist)
f.write(os.linesep)
else:
break
f.close
f = open('proxies.txt', 'r')
x = f.readlines()
f.close
os.remove('proxies.txt')
list2 = sorted(set(x), key=x.index)
for proxy in list2:
with open('proxies.txt', 'a+') as f:
f.writelines(proxy)
f.close