My python proxy scraper with multiprocessing

proxy_scraper_2.py

'''
A threaded proxy scraper

This script will scrape a list of websites for ip:port pairs.
'''

import argparse
import file_handler
from strip import strip_proxies
from scrape import run_scraper
from multiprocessing import Process, Queue, JoinableQueue, cpu_count, Event
from selenium import webdriver
from selenium.common.exceptions import WebDriverException, NoSuchElementException

def main():
    # Initialize the argparser
    parser = argparse.ArgumentParser()
    parser.add_argument('sitelist', help='A CSV file containing a list of sites to scrape.')
    parser.add_argument('-j', '--jobs', help='The number of jobs to spawn while scraping.')
    parser.add_argument('-o', '--output', help='The name of the file that the proxylist is saved to')
    arguments = parser.parse_args()

    # Initialize the queues
    scrapeQueue = JoinableQueue()
    dumpQueue = Queue()

    # Set Options
    if arguments.jobs:
        try:
            numThreads = int(arguments.jobs)
        except ValueError:
            print "--jobs must be a whole number"
            return
    else:
        try:
            numThreads = cpu_count()
        except:
            numThreads = 1

    if arguments.output:
        try:
            out = str(arguments.output)
        except:
            print 'Invalid output file, defaulting to proxy.txt'
            out = 'proxy.txt'
    else:
        out = 'proxy.txt'


    # Create empty proxy list
    proxylist = []

    # Start the scrapers
    for i in range(numThreads):
        worker = Process(target=ScrapeJob, args=(scrapeQueue, dumpQueue))
        worker.start()

    #Load the scrapeQueue
    with open(arguments.sitelist, 'rb') as FILE:
        file_handler.load_queue(scrapeQueue, FILE)

    #Wait for the scrapers to finish
    scrapeQueue.join()

    #Regex the ip port pairs out of the textDumps
    while not dumpQueue.empty():
        textDump = dumpQueue.get()
        strip_proxies(textDump, proxylist)

    #write it all to a file
    with open(out, 'wb') as FILE:
        file_handler.save_list_to_file(proxylist, FILE)
        FILE.close()
    return

class ScrapeJob(Process):
    def __init__(self, scrape_queue, dump_queue):
        Process.__init__(self)
        self.exit = Event()
        self.driver = webdriver.Firefox()
        self.scrape_queue = scrape_queue
        self.dump_queue = dump_queue
        self.run()

    def run(self):
        while not self.scrape_queue.empty():
            job_args = self.scrape_queue.get()
            name = job_args.pop(0)
            try:
                print 'Scraping ' + name
                run_scraper(self.driver, self.dump_queue, *job_args)
                print '[+] ' + name + ' has been scraped.'
            except WebDriverException as e:
                print ('[-] ' + name + ' WebDriver failed with exception: \n' +
                       str(e))
            except NoSuchElementException as e:
                print ('[-] ' + name +
                       ' Scraper failed with NoSuchElementException: \n' + str(e))
            except:
                print '[-] ' + name + ': Unknown Exception.'
            self.scrape_queue.task_done()
        self.shutdown()

    def shutdown(self):
        self.driver.close()
        self.exit.set()

if __name__ == '__main__':
    main()

==============================
scrape.py

'''
This script takes a selenium webdriver and scrapes the specified site. The full text of the site is dumped to the provided queue for later processing.
'''
def run_scraper(driver, queue, url, urlExtension='',
                startPage=None, endPage=None, fill=None):
    if startPage == None or endPage == None:
        scrape_single_page(driver, queue, url)
    else:
        scrape_multiple_pages(driver, queue, url, urlExtension,
                              startPage, endPage, fill)

def scrape_single_page(driver, queue, url):
    driver.get(url)
    dump_text(driver, queue)

def scrape_multiple_pages(driver, queue, url, urlExtension,
                          startPage=None, endPage=None, fill=None):
    for i in range(int(startPage), int(endPage) + 1):
        if fill == None:
            driver.get(url + str(i) + urlExtension)
        else:
            driver.get(url + str(i).zfill(int(fill)) + urlExtension)
        dump_text(driver, queue)

def dump_text(driver, queue):
    dump = driver.find_element_by_tag_name('body')
    queue.put(dump.text)

==============================
strip.py

'''
This module pulls ip port pairs from a string of text and adds them to the provided list.
'''

import re

def strip_proxies(text, proxylist):
    #Setup the regular expressions to find the pairs
    ip_colon_port = re.compile('\d+\.\d+\.\d+\.\d+\:\d+')
    ip_space_port = re.compile('\d+\.\d+\.\d+\.\d+\s\d+')

    #Add everything that matches the regex to the proxylist
    proxylist.extend(ip_colon_port.findall(text))
    proxylist.extend(ip_space_port.findall(text))

    #Convert to ASCII then replace spaces with colons
    proxylist = [convert_to_ascii(proxy) for proxy in proxylist]
    proxylist = [replace_with_colon(proxy) for proxy in proxylist]

def convert_to_ascii(text):
    return text.encode('ascii', 'ignore')

def replace_with_colon(text):
    return text.replace(' ', ':')

==============================
file_handler.py

import csv

def load_queue(queue, FILE):
    reader = csv.reader(FILE)
    for row in reader:
        if row and not ','.join(row).startswith('#'):
            queue.put(row)

def load_list(the_list, FILE):
    reader = csv.reader(FILE)
    for row in reader:
        if row and not ','.join(row).startswith('#'):
            list.extend(row)

def save_list_to_file(the_list, FILE):
        for string in the_list:
            FILE.write(string + "\n")