# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html

import random
import time
import logging as logger

import scrapy
from redisbloom.client import Client
from scrapy import signals
from scrapy.exceptions import IgnoreRequest
from twisted.internet.error import ConnectionRefusedError

from AirplaneSpiders.utils import http_get


class MyProxyDownloaderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.
        if self.request_count == 0 or self.request_count >= int(spider.settings['PER_BATCH_IP_USE_TIMES']):
            self.request_count = 0
            self.proxyIpList = []
            proxy_service = spider.settings['PROXY_SERVICE']
            #调用自建Ip池api
            self.proxyIpList = http_get(proxy_service + 'get_all')
            while len(self.proxyIpList) == 0:
                self.proxyIpList = http_get(proxy_service + 'get_all')
                time.sleep(1)

        proxy = random.choice(self.proxyIpList)
        logger.info("使用代理IP：{}进行访问".format(proxy['proxy']))

        if 'splash' in request.meta:
            request.meta['splash']['args']['proxy'] = 'http://%s' % proxy['proxy']
        else:
            request.meta['proxy'] = 'http://%s' % proxy['proxy']

        self.request_count += 1
        return None

    def process_exception(self, request, exception, spider):
        if isinstance(exception, ConnectionRefusedError):
            return request
        return None

    def spider_opened(self, spider):
        self.request_count = 0
        spider.logger.info('Spider opened: %s' % spider.name)


class UserAgentDownloaderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        s.uas = crawler.spider.settings['CUSTOM_USER_AGENT']
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        UA = random.choice(self.uas)
        if request.headers is None:
            request.headers = {}
        request.headers['User-Agent'] = UA
        return None

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class DumpFilterDownloaderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        redisHost = crawler.spider.settings['REDIS_HOST']
        redisPort = crawler.spider.settings['REDIS_PORT']
        pwd = crawler.spider.settings['REDIS_PWD']
        s.bloomFilter = Client(host=redisHost, port=redisPort, password=pwd)
        s.deepth = crawler.spider.settings['CRAWL_DEEPTH']
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        #判重
        if 'CUSTOM_FILTER_KEY' in request.meta and 'CUSTOM_FILTER_VAL' in request.meta:
            key = request.meta['CUSTOM_FILTER_KEY']
            val = request.meta['CUSTOM_FILTER_VAL']
            if self.bloomFilter.bfExists(key, val) == 1:
                raise IgnoreRequest()
        return None

    def process_exception(self, request, exception, spider):
        return None

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class DumpFiliterSpiderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        redisHost = crawler.spider.settings['REDIS_HOST']
        redisPort = crawler.spider.settings['REDIS_PORT']
        pwd = crawler.spider.settings['REDIS_PWD']
        s.bloomFilter = Client(host=redisHost, port=redisPort, password=pwd)
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_output(self, response, result, spider):
        for i in result:
            #对于符合爬取条件的请求，加入判重
            if isinstance(i, scrapy.Item):
                if 'CUSTOM_FILTER_KEY' in response.request.meta and 'CUSTOM_FILTER_VAL' in response.request.meta:
                    key = response.request.meta['CUSTOM_FILTER_KEY']
                    val = response.request.meta['CUSTOM_FILTER_VAL']
                    if self.bloomFilter.bfAdd(key, val) <= 0:
                        continue
            yield i

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)
