# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html

import random

import scrapy
from redisbloom.client import Client
from scrapy import signals
from scrapy.exceptions import IgnoreRequest


class UserAgentDownloaderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        s.uas = crawler.spider.settings['CUSTOM_USER_AGENT']
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        UA = random.choice(self.uas)
        if request.headers is None:
            request.headers = {}
        request.headers['User-Agent'] = UA
        return None

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class DumpFilterDownloaderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        redis_host = crawler.spider.settings['REDIS_HOST']
        redis_port = crawler.spider.settings['REDIS_PORT']
        pwd = crawler.spider.settings['REDIS_PWD']
        s.bloomFilter = Client(host=redis_host, port=redis_port, password=pwd)
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s

    def process_request(self, request, spider):
        if 'CUSTOM_FILTER_KEY' in request.meta and 'CUSTOM_FILTER_VAL' in request.meta:
            key = request.meta['CUSTOM_FILTER_KEY']
            val = request.meta['CUSTOM_FILTER_VAL']
            if self.bloomFilter.bfExists(key, val) == 1:
                spider.logger.info('检测到重复 URL，已过滤：%s' % val)
                raise IgnoreRequest()
        return None

    def process_exception(self, request, exception, spider):
        return None

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

    def spider_closed(self, spider):
        self.bloomFilter.close()
        spider.logger.info('Spider closed: %s' % spider.name)


class DumpFilterSpiderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        s = cls()
        redis_host = crawler.spider.settings['REDIS_HOST']
        redis_port = crawler.spider.settings['REDIS_PORT']
        pwd = crawler.spider.settings['REDIS_PWD']
        s.bloomFilter = Client(host=redis_host, port=redis_port, password=pwd)
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s

    def process_spider_output(self, response, result, spider):
        for i in result:
            if isinstance(i, scrapy.Item):
                if 'CUSTOM_FILTER_KEY' in response.request.meta and 'CUSTOM_FILTER_VAL' in response.request.meta:
                    key = response.request.meta['CUSTOM_FILTER_KEY']
                    val = response.request.meta['CUSTOM_FILTER_VAL']
                    if i['es_carriertype'] != 'news':
                        val = i['es_urlname']
                    if self.bloomFilter.bfAdd(key, val) <= 0:
                        spider.logger.info('检测到重复URL，已过滤：%s' % val)
                        continue
            yield i

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

    def spider_closed(self, spider):
        self.bloomFilter.close()
        spider.logger.info('Spider closed: %s' % spider.name)
