book/epub-crawler
1
0
Fork 0
mirror of https://github.com/apachecn/epub-crawler.git synced 2025-06-06 17:56:57 +00:00
Code Issues Releases Wiki Activity

2022-03-15 23:33:14

Browse Source
This commit is contained in:
wizardforcel 2022-03-15 23:33:14 +08:00
parent c1a08f7b62
commit 362c34c283
1 changed files with 17 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
EpubCrawler/img.py
View File

@ -25,21 +25,26 @@ def get_img_src(el_img):
if url: break
return url
def tr_download_img(url, imgs, picname):
def tr_download_img_safe(url, imgs, picname):
try:
data = request_retry(
'GET', url,
headers=config['headers'],
check_status=config['checkStatus'],
retry=config['retry'],
timeout=config['timeout'],
proxies=config['proxy'],
).content
data = opti_img(data, config['optiMode'], config['colors'])
imgs[picname] = data or b''
time.sleep(config['wait'])
tr_download_img(url, imgs, picname)
except Exception as ex:
print(ex)
imgs[picname] = b''
def tr_download_img(url, imgs, picname):
data = request_retry(
'GET', url,
headers=config['headers'],
check_status=config['checkStatus'],
retry=config['retry'],
timeout=config['timeout'],
proxies=config['proxy'],
).content
data = opti_img(data, config['optiMode'], config['colors'])
imgs[picname] = data or b''
time.sleep(config['wait'])
def process_img_data_url(url, el_img, imgs, **kw):
if not re.search(RE_DATA_URL, url):