# -*- coding: UTF-8 -*-
import requests
import os
import subprocess
import sys
from tqdm import tqdm
import time,random
from HTMLParser import HTMLParser
import sys
reload(sys)
sys.setdefaultencoding('utf8')
class WebRequest(object):
def __init__(self, *args, **kwargs):
pass
@property
def user_agent(self):
"""
return an User-Agent at random
:return:
"""
ua_list = [
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71',
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
]
return random.choice(ua_list)
@property
def header(self):
"""
basic header
:return:
"""
return {'User-Agent': self.user_agent,
'Accept': '*/*',
'Connection': 'keep-alive'
}
def get(self, url, header=None, retry_time=5, timeout=30,
retry_flag=list(), retry_interval=5, *args, **kwargs):
"""
get method
:param url: target url
:param header: headers
:param retry_time: retry time when network error
:param timeout: network timeout
:param retry_flag: if retry_flag in content. do retry
:param retry_interval: retry interval(second)
:param args:
:param kwargs:
:return:
"""
headers = self.header
if header and isinstance(header, dict):
headers.update(header)
while True:
try:
html = requests.get(url, headers=headers, timeout=timeout, stream=True)
print 'content size: %d' % len(html.content)
if any(f in html.content for f in retry_flag):
raise Exception
if not html.content:
print 'content is Null,retry...~' + url
raise Exception
return html
except Exception as e:
print(e)
retry_time -= 1
if retry_time <= 0:
return requests.get("https://www.google.com/")
time.sleep(retry_interval)
except exceptions.Timeout as e:
print(e)
time.sleep(retry_interval)
except exceptions.ConnectionError as e:
print(e)
time.sleep(retry_interval)
except exceptions.ConnectTimeout as e:
print(e)
time.sleep(retry_interval)
except exceptions.ReadTimeout as e:
print(e)
time.sleep(retry_interval)
def DownloadAudio(url, path, refer=None):
with open(path, 'wb') as handle:
if refer:
hdr = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
'Referer': refer}
else:
hdr = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
'Accept-Encoding': 'none',
'Accept-Language': 'en-US,en;q=0.8',
'Connection': 'keep-alive'}
wr = WebRequest()
#print hdr
response = wr.get(url,header=hdr)
#html = requests.get('https://www.example.com', proxies={"http": "http://{}".format(proxy)})
#response = requests.get(url, stream=True, headers={'User-agent': 'Mozilla/5.0'})
#print url
for block in tqdm(response.iter_content(),ascii=True, desc='DownloadAudio'):
if not block:
break
handle.write(block)
def translate(to_translate, to_langage="auto", langage="auto"):
'''Return the translation using google translate
you must shortcut the langage you define (French = fr, English = en, Spanish = es, etc...)
if you don't define anything it will detect it or use english by default
Example:
print(translate("salut tu vas bien?", "en"))
hello you alright?'''
agents = {'User-Agent':"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)"}
before_trans = 'class="t0">'
link = "http://translate.google.com/m?ie=UTF-8&hl=%s&sl=%s&q=%s" % (to_langage, langage, to_translate.replace(" ", "+"))
request = requests.get(link, headers=agents)
page = request.content
#print page
result = page[page.find(before_trans)+len(before_trans):]
result = result.split("<")[0]
return result
tsxt_sen = []
from nltk.tokenize import sent_tokenize
def sentence_split(sentence):
text_sen = []
for s in sentence.split(','):
# if '?' in s:
# tsxt_sen.extend(s.split('?'))
# elif ',' in s:
# tsxt_sen.extend(s.split(','))
# else:
text_sen.append(s)
return tsxt_sen
str = '''
I have a secret to tell you. Come close. Closer.
You don’t have to pay a lot of money to redo your bedroom.
It's way too easy to spend hundreds of dollars buying new blankets, sheets, and pillows from luxury brands, but please don’t. Seriously, Amazon has some genuinely amazing bedding products that are all reasonably priced and loved by literally thousands of people. So whether you’re looking for a new duvet, pillowcases, or whatever, it's worth checking Amazon first.
In case you’re skeptical, here are 18 bedding products with more than 1,000 reviews from happy, happy Amazon customers.
'''
text = ''.join(str).strip().lstrip().rstrip().replace('\n',' ')
with open('./text.txt', 'wb') as handle:
handle.write(text)
str = str.replace('”','"')
str = str.replace('“','"')
def splitStr(str):
arr = str.split(' ')
sz = len(arr)
return [' '.join(arr[0:sz/2]),' '.join(arr[(sz/2):])]
#print splitStr('with wording from North Korea about getting rid of its nuclear weapons and a guarantee from the United States that it would not interfere with the North’s regime or demand redress for human rights abuses.')
keyArr = []
sent_tokenize_list = sent_tokenize(str)
for s in sent_tokenize_list:
sz = len(' '.join(s.split()))
if sz <= 190:
keyArr.append(' '.join(s.split()))
#print '[ '+' '.join(s.split())+' ]'
else:
for ses in s.split(','):
if len(ses) > 190:
keyArr.extend(splitStr(ses))
else:
keyArr.append(' '.join(ses.split()))
API_URL = "http://translate.google.com/translate_tts?ie=UTF-8&tl=en-us"
#key = HTMLParser.unescape.__func__(HTMLParser, translate(str,'en'))
#keyArr = sentence_split(str)
#print([text[i:i+n] for i in xrange(0, len(text), n)])
os.system('rm ./*.mp3')
namei = 0
for i in keyArr:
juzi = ' '.join(i.split())
sz = len(juzi)
link = API_URL+"&q="+juzi+"&client=tw-ob"
print link
path = "./%d.mp3" % namei
#print path
DownloadAudio(link,path)
os.system('file '+path)
time.sleep(1)
namei = namei + 1
os.system('cat ./*.mp3 >> full.mp3')