82 lines
2.5 KiB
Python
82 lines
2.5 KiB
Python
|
# -*- coding: utf-8 -*-
|
|||
|
|
"""
|
|||
|
|
-------------------------------------------------
|
|||
|
|
File Name： RawProxyCheck
|
|||
|
|
Description : check raw_proxy to useful
|
|||
|
|
Author : JHao
|
|||
|
|
date： 2019/8/6
|
|||
|
|
-------------------------------------------------
|
|||
|
|
Change Activity:
|
|||
|
|
2019/8/6: check raw_proxy to useful
|
|||
|
|
-------------------------------------------------
|
|||
|
|
"""
|
|||
|
|
__author__ = 'JHao'
|
|||
|
|
|
|||
|
|
from threading import Thread
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
from Queue import Empty, Queue # py2
|
|||
|
|
except:
|
|||
|
|
from queue import Empty, Queue # py3
|
|||
|
|
|
|||
|
|
from Util import LogHandler
|
|||
|
|
from Manager import ProxyManager
|
|||
|
|
from ProxyHelper import Proxy, checkProxyUseful
|
|||
|
|
|
|||
|
|
|
|||
|
|
class RawProxyCheck(ProxyManager, Thread):
|
|||
|
|
def __init__(self, queue, thread_name):
|
|||
|
|
ProxyManager.__init__(self)
|
|||
|
|
Thread.__init__(self, name=thread_name)
|
|||
|
|
self.log = LogHandler('raw_proxy_check')
|
|||
|
|
self.queue = queue
|
|||
|
|
|
|||
|
|
def run(self):
|
|||
|
|
self.log.info("RawProxyCheck - {} : start".format(self.name))
|
|||
|
|
self.db.changeTable(self.useful_proxy_queue)
|
|||
|
|
while True:
|
|||
|
|
try:
|
|||
|
|
proxy_json = self.queue.get(block=False)
|
|||
|
|
except Empty:
|
|||
|
|
self.log.info("RawProxyCheck - {} : exit".format(self.name))
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
proxy_obj = Proxy.newProxyFromJson(proxy_json)
|
|||
|
|
|
|||
|
|
proxy_obj, status = checkProxyUseful(proxy_obj)
|
|||
|
|
if status:
|
|||
|
|
if self.db.exists(proxy_obj.proxy):
|
|||
|
|
self.log.info('RawProxyCheck - {} : {} validation exists'.format(self.name,
|
|||
|
|
proxy_obj.proxy.ljust(20)))
|
|||
|
|
else:
|
|||
|
|
self.db.put(proxy_obj)
|
|||
|
|
self.log.info(
|
|||
|
|
'RawProxyCheck - {} : {} validation pass'.format(self.name, proxy_obj.proxy.ljust(20)))
|
|||
|
|
else:
|
|||
|
|
self.log.info('RawProxyCheck - {} : {} validation fail'.format(self.name, proxy_obj.proxy.ljust(20)))
|
|||
|
|
self.queue.task_done()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def doRawProxyCheck():
|
|||
|
|
proxy_queue = Queue()
|
|||
|
|
|
|||
|
|
pm = ProxyManager()
|
|||
|
|
pm.db.changeTable(pm.raw_proxy_queue)
|
|||
|
|
for _proxy in pm.db.getAll():
|
|||
|
|
proxy_queue.put(_proxy)
|
|||
|
|
pm.db.clear()
|
|||
|
|
|
|||
|
|
thread_list = list()
|
|||
|
|
for index in range(20):
|
|||
|
|
thread_list.append(RawProxyCheck(proxy_queue, "thread_%s" % index))
|
|||
|
|
|
|||
|
|
for thread in thread_list:
|
|||
|
|
thread.start()
|
|||
|
|
|
|||
|
|
for thread in thread_list:
|
|||
|
|
thread.join()
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == '__main__':
|
|||
|
|
doRawProxyCheck()
|