# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
from urllib import parse

from scrapy import signals
import random
from ForumSpiders.utils.time_util import pause_for_random
from redisbloom.client import Client
import scrapy


class UserAgentDownloaderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        s.uas = crawler.spider.settings['CUSTOM_USER_AGENT']
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        ua = random.choice(self.uas)
        if request.headers is None:
            request.headers = {}

        request.headers['User-Agent'] = ua
        request.headers['Host'] = parse.urlparse(request.url)[1]

        return None

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s. UserAgentDownloaderMiddleware started' % spider.name)


class RandomDelayDownloaderMiddleware(object):
    def __init__(self, delay):
        self.delay = delay

    @classmethod
    def from_crawler(cls, crawler):
        delay = crawler.spider.settings.get("RANDOM_DELAY", 10)
        if not isinstance(delay, int):
            raise ValueError("RANDOM_DELAY need a int")
        return cls(delay)

    def process_request(self, request, spider):
        spider.logger.debug("# random delay: %s s #" % self.delay)
        pause_for_random(self.delay)


class DumpFilterSpiderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        s = cls()
        redis_host = crawler.spider.settings['REDIS_HOST']
        redis_port = crawler.spider.settings['REDIS_PORT']
        redis_pwd = crawler.spider.settings['REDIS_PWD']
        s.bloomFilter = Client(host=redis_host, port=redis_port, password=redis_pwd)
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s

    def process_spider_output(self, response, result, spider):
        for i in result:
            # 对于符合爬取条件的请求，加入判重
            if isinstance(i, scrapy.Item):
                key = 'ForumSpiders:ITEM_FILTER'
                val = i['es_urlname']
                if self.bloomFilter.bfAdd(key, val) <= 0:
                    continue
            yield i

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

    def spider_closed(self, spider):
        self.bloomFilter.close()
        spider.logger.info('Spider closed: %s' % spider.name)
