70 lines
2.2 KiB
Python
70 lines
2.2 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""
|
||
-------------------------------------------------
|
||
File Name： CheckProxy
|
||
Description : used for check getFreeProxy.py
|
||
Author : JHao
|
||
date： 2018/7/10
|
||
-------------------------------------------------
|
||
Change Activity:
|
||
2018/7/10: CheckProxy
|
||
-------------------------------------------------
|
||
"""
|
||
__author__ = 'JHao'
|
||
|
||
from getFreeProxy import GetFreeProxy
|
||
from Util.utilFunction import verifyProxyFormat
|
||
|
||
|
||
from Util.LogHandler import LogHandler
|
||
|
||
log = LogHandler('check_proxy', file=False)
|
||
|
||
|
||
class CheckProxy(object):
|
||
|
||
@staticmethod
|
||
def checkAllGetProxyFunc():
|
||
"""
|
||
检查getFreeProxy所有代理获取函数运行情况
|
||
Returns:
|
||
None
|
||
"""
|
||
import inspect
|
||
member_list = inspect.getmembers(GetFreeProxy, predicate=inspect.isfunction)
|
||
proxy_count_dict = dict()
|
||
for func_name, func in member_list:
|
||
log.info(u"开始运行 {}".format(func_name))
|
||
try:
|
||
proxy_list = [_ for _ in func() if verifyProxyFormat(_)]
|
||
proxy_count_dict[func_name] = len(proxy_list)
|
||
except Exception as e:
|
||
log.info(u"代理获取函数 {} 运行出错!".format(func_name))
|
||
log.error(str(e))
|
||
log.info(u"所有函数运行完毕 " + "***" * 5)
|
||
for func_name, func in member_list:
|
||
log.info(u"函数 {n}, 获取到代理数: {c}".format(n=func_name, c=proxy_count_dict.get(func_name, 0)))
|
||
|
||
@staticmethod
|
||
def checkGetProxyFunc(func):
|
||
"""
|
||
检查指定的getFreeProxy某个function运行情况
|
||
Args:
|
||
func: getFreeProxy中某个可调用方法
|
||
|
||
Returns:
|
||
None
|
||
"""
|
||
func_name = getattr(func, '__name__', "None")
|
||
log.info("start running func: {}".format(func_name))
|
||
count = 0
|
||
for proxy in func():
|
||
if verifyProxyFormat(proxy):
|
||
log.info("{} fetch proxy: {}".format(func_name, proxy))
|
||
count += 1
|
||
log.info("{n} completed, fetch proxy number: {c}".format(n=func_name, c=count))
|
||
|
||
|
||
if __name__ == '__main__':
|
||
CheckProxy.checkAllGetProxyFunc()
|
||
CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy01) |