Advance proxy scraping script

Joined
May 19, 2019
Messages
10
Reaction score
3
As you guys asked here we go with my new advance proxy scraping script,if you have any questions just feel free to ask .

Code:
from bs4 import BeautifulSoup
import cfscrape
from pathlib import Path
import os
import time

repeat = 2  # scrape 2 times
time_interval = 10  # seconds between each repeat

while True:
    if repeat >= 0:
        repeat -= 1
        time.sleep(time_interval)
        url = "https://hidemyna.me/en/proxy-list/"
        scraper = cfscrape.create_scraper()
        html = scraper.get(url).content
        soup = BeautifulSoup(html, 'lxml')
        table = soup.find("tbody")
        for listt in table:
            for proxy in listt.findAll("td")[0:1]:
                for port in listt.findAll("td")[1:2]:
                    # print(proxy, port)
                    proxylist = f"{proxy.text}:{port.text}"
                    with open('proxies.txt', 'a+') as f:
                        f.writelines(proxylist)
                        f.write(os.linesep)
    else:
        break

f.close
f = open('proxies.txt', 'r')
x = f.readlines()
f.close
os.remove('proxies.txt')

list2 = sorted(set(x), key=x.index)

for proxy in list2:
    with open('proxies.txt', 'a+') as f:
        f.writelines(proxy)
f.close
 
thanks... has this been clearing Cloudflare as is? Any advice for if it hits recaptchas?
 
Back
Top