96 lines
2.5 KiB
Python
96 lines
2.5 KiB
Python
|
# -*- coding: utf-8 -*-
|
|||
|
|
# !/usr/bin/env python
|
|||
|
|
"""
|
|||
|
|
-------------------------------------------------
|
|||
|
|
File Name： utilFunction.py
|
|||
|
|
Description : tool function
|
|||
|
|
Author : JHao
|
|||
|
|
date： 2016/11/25
|
|||
|
|
-------------------------------------------------
|
|||
|
|
Change Activity:
|
|||
|
|
2016/11/25: 添加robustCrawl、verifyProxy、getHtmlTree
|
|||
|
|
-------------------------------------------------
|
|||
|
|
"""
|
|||
|
|
import requests
|
|||
|
|
from lxml import etree
|
|||
|
|
|
|||
|
|
from Util.WebRequest import WebRequest
|
|||
|
|
|
|||
|
|
|
|||
|
|
def robustCrawl(func):
|
|||
|
|
def decorate(*args, **kwargs):
|
|||
|
|
try:
|
|||
|
|
return func(*args, **kwargs)
|
|||
|
|
except Exception as e:
|
|||
|
|
pass
|
|||
|
|
# logger.info(u"sorry, 抓取出错。错误原因:")
|
|||
|
|
# logger.info(e)
|
|||
|
|
|
|||
|
|
return decorate
|
|||
|
|
|
|||
|
|
|
|||
|
|
def verifyProxyFormat(proxy):
|
|||
|
|
"""
|
|||
|
|
检查代理格式
|
|||
|
|
:param proxy:
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
import re
|
|||
|
|
verify_regex = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}"
|
|||
|
|
_proxy = re.findall(verify_regex, proxy)
|
|||
|
|
return True if len(_proxy) == 1 and _proxy[0] == proxy else False
|
|||
|
|
|
|||
|
|
|
|||
|
|
def getHtmlTree(url, **kwargs):
|
|||
|
|
"""
|
|||
|
|
获取html树
|
|||
|
|
:param url:
|
|||
|
|
:param kwargs:
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
header = {'Connection': 'keep-alive',
|
|||
|
|
'Cache-Control': 'max-age=0',
|
|||
|
|
'Upgrade-Insecure-Requests': '1',
|
|||
|
|
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko)',
|
|||
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
|||
|
|
'Accept-Encoding': 'gzip, deflate, sdch',
|
|||
|
|
'Accept-Language': 'zh-CN,zh;q=0.8',
|
|||
|
|
}
|
|||
|
|
# TODO 取代理服务器用代理服务器访问
|
|||
|
|
wr = WebRequest()
|
|||
|
|
html = wr.get(url=url, header=header).content
|
|||
|
|
return etree.HTML(html)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def tcpConnect(proxy):
|
|||
|
|
"""
|
|||
|
|
TCP 三次握手
|
|||
|
|
:param proxy:
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
from socket import socket, AF_INET, SOCK_STREAM
|
|||
|
|
s = socket(AF_INET, SOCK_STREAM)
|
|||
|
|
ip, port = proxy.split(':')
|
|||
|
|
result = s.connect_ex((ip, int(port)))
|
|||
|
|
return True if result == 0 else False
|
|||
|
|
|
|||
|
|
|
|||
|
|
def validUsefulProxy(proxy):
|
|||
|
|
"""
|
|||
|
|
检验代理是否可用
|
|||
|
|
:param proxy:
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
if isinstance(proxy, bytes):
|
|||
|
|
proxy = proxy.decode("utf8")
|
|||
|
|
proxies = {"http": "http://{proxy}".format(proxy=proxy)}
|
|||
|
|
try:
|
|||
|
|
r = requests.get('http://www.baidu.com', proxies=proxies, timeout=10, verify=False)
|
|||
|
|
if r.status_code == 200:
|
|||
|
|
return True
|
|||
|
|
except Exception as e:
|
|||
|
|
pass
|
|||
|
|
return False
|
|||
|
|
|