114 lines
3.8 KiB
Python
114 lines
3.8 KiB
Python
# -*- coding: utf-8 -*-
|
||
# !/usr/bin/env python
|
||
"""
|
||
-------------------------------------------------
|
||
File Name： ProxyManager.py
|
||
Description :
|
||
Author : JHao
|
||
date： 2016/12/3
|
||
-------------------------------------------------
|
||
Change Activity:
|
||
2016/12/3:
|
||
-------------------------------------------------
|
||
"""
|
||
__author__ = 'JHao'
|
||
|
||
import random
|
||
import json
|
||
from ProxyHelper import Proxy
|
||
from DB.DbClient import DbClient
|
||
from Config.ConfigGetter import config
|
||
from Util.LogHandler import LogHandler
|
||
from Util.utilFunction import verifyProxyFormat
|
||
from ProxyGetter.getFreeProxy import GetFreeProxy
|
||
|
||
|
||
class ProxyManager(object):
|
||
"""
|
||
ProxyManager
|
||
"""
|
||
|
||
def __init__(self):
|
||
self.db = DbClient()
|
||
self.raw_proxy_queue = 'raw_proxy'
|
||
self.log = LogHandler('proxy_manager')
|
||
# self.useful_proxy_queue = 'ProxyPool:useful_proxy_3'
|
||
self.useful_proxy_queue = 'ProxyPool:useful_proxy_63'
|
||
|
||
def fetch(self):
|
||
"""
|
||
fetch proxy into db by ProxyGetter
|
||
:return:
|
||
"""
|
||
self.db.changeTable(self.raw_proxy_queue)
|
||
proxy_set = set()
|
||
self.log.info("ProxyFetch : start")
|
||
for proxyGetter in config.proxy_getter_functions:
|
||
self.log.info("ProxyFetch - {func}: start".format(func=proxyGetter))
|
||
try:
|
||
for proxy_str in getattr(GetFreeProxy, proxyGetter.strip())():
|
||
proxy_dict = json.loads(proxy_str, encoding='utf-8')
|
||
proxy = proxy_dict['proxy']
|
||
proxy_type = proxy_dict['proxy_type']
|
||
anonimity = proxy_dict['anonimity']
|
||
#self.log.info(anonimity)
|
||
|
||
if not proxy or not verifyProxyFormat(proxy):
|
||
self.log.error('ProxyFetch - {func}: '
|
||
'{proxy} illegal'.format(func=proxyGetter, proxy=proxy.ljust(20)))
|
||
continue
|
||
elif proxy in proxy_set:
|
||
self.log.info('ProxyFetch - {func}: '
|
||
'{proxy} exist'.format(func=proxyGetter, proxy=proxy.ljust(20)))
|
||
continue
|
||
else:
|
||
self.db.put(Proxy(proxy, source=proxyGetter, proxy_type=proxy_type, anonimity=anonimity))
|
||
proxy_set.add(proxy)
|
||
self.log.info('ProxyFetch - {func}: '
|
||
'{proxy} added'.format(func=proxyGetter, proxy=proxy.ljust(20)))
|
||
except Exception as e:
|
||
self.log.error("ProxyFetch - {func}: error".format(func=proxyGetter))
|
||
self.log.error(str(e))
|
||
|
||
def get(self):
|
||
"""
|
||
return a useful proxy
|
||
:return:
|
||
"""
|
||
self.db.changeTable(self.useful_proxy_queue)
|
||
item_list = self.db.getAll()
|
||
if item_list:
|
||
random_choice = random.choice(item_list)
|
||
return Proxy.newProxyFromJson(random_choice)
|
||
return None
|
||
|
||
def delete(self, proxy_str):
|
||
"""
|
||
delete proxy from pool
|
||
:param proxy_str:
|
||
:return:
|
||
"""
|
||
self.db.changeTable(self.useful_proxy_queue)
|
||
self.db.delete(proxy_str)
|
||
|
||
def getAll(self):
|
||
"""
|
||
get all proxy from pool as list
|
||
:return:
|
||
"""
|
||
self.db.changeTable(self.useful_proxy_queue)
|
||
item_list = self.db.getAll()
|
||
return [Proxy.newProxyFromJson(_) for _ in item_list]
|
||
|
||
def getNumber(self):
|
||
self.db.changeTable(self.raw_proxy_queue)
|
||
total_raw_proxy = self.db.getNumber()
|
||
self.db.changeTable(self.useful_proxy_queue)
|
||
total_useful_queue = self.db.getNumber()
|
||
return {'raw_proxy': total_raw_proxy, 'useful_proxy': total_useful_queue}
|
||
|
||
|
||
if __name__ == '__main__':
|
||
pp = ProxyManager()
|
||
pp.fetch()
|