# -*- coding: utf-8 -*-

import json
import logging
import random
import re
import time

import redis
import scrapy
from redisbloom.client import Client
from scrapy import signals
from scrapy.exceptions import IgnoreRequest
from scrapy.utils.project import get_project_settings
from simhash import Simhash

from CustomWebsite.scrapy_selenium import SeleniumRequest
from CustomWebsite.utils.http_utils import http_get
from CustomWebsite.utils.http_utils import http_post
from CustomWebsite.utils.string_utils import check_keywords

project_settings = get_project_settings()


class AutoProxyDownloaderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.
        if 'parse_rule' not in request.meta:
            return None
        if request.meta['parse_rule']['proxyMethod'] == '1':
            # 使用动态代理(ip池)
            if self.request_count == 0 or self.request_count >= int(spider.settings['PER_BATCH_IP_USE_TIMES']):
                self.request_count = 0
                self.proxy_ip_list = []
                proxy_service = spider.settings['PROXY_SERVICE']
                # 调用自建Ip池api
                self.proxy_ip_list = json.loads(http_get(proxy_service + '/get_all').text)
                while len(self.proxy_ip_list) == 0:
                    self.proxy_ip_list = json.loads(http_get(proxy_service + '/get_all').text)
                    time.sleep(1)
            proxy = random.choice(self.proxy_ip_list)
            self.request_count += 1
            if 'splash' in request.meta:
                request.meta['splash']['args']['proxy'] = 'http://%s' % proxy['proxy']
            else:
                request.meta['proxy'] = 'http://%s' % proxy['proxy']
        return None

    def process_exception(self, request, exception, spider):
        # if isinstance(exception,ConnectionRefusedError):
        #     proxy_service = spider.settings['PROXY_SERVICE']
        #     http_get(proxy_service + 'delete?proxy=' + request.meta['proxy'].split('/')[-1])
        #     return request
        return None

    def spider_opened(self, spider):
        self.request_count = 0
        spider.logger.info('Spider opened: %s' % spider.name)


class UserAgentDownloaderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        s.uas = crawler.spider.settings['CUSTOM_USER_AGENT']
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        UA = random.choice(self.uas)
        if request.headers is None:
            request.headers = {}
        request.headers['User-Agent'] = UA
        return None

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class DeepLimitDownloaderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        # s.depth = crawler.spider.settings['CRAWL_DEPTH']
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # 爬取深度限制
        if 'parse_rule' not in request.meta:
            return None
        site_depth = int(request.meta['parse_rule']['depth'])
        if site_depth <= 0:
            site_depth = 2
        if request.meta['parse_rule']['current_depth'] > site_depth:
            raise IgnoreRequest()
        return None

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class DumpFilterDownloaderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        redis_host = crawler.spider.settings['REDIS_HOST']
        redis_port = crawler.spider.settings['REDIS_PORT']
        pwd = crawler.spider.settings['REDIS_PWD']
        s.bloomFilter = Client(host=redis_host, port=redis_port, password=pwd)
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s

    def process_request(self, request, spider):
        # 判重
        if 'CUSTOM_FILTER_KEY' in request.meta and 'CUSTOM_FILTER_VAL' in request.meta:
            key = request.meta['CUSTOM_FILTER_KEY']
            val = request.meta['CUSTOM_FILTER_VAL']
            if self.bloomFilter.bfExists(key, val) == 1:
                spider.logger.debug('检测到重复 URL，已过滤：%s' % val)
                raise IgnoreRequest()
        return None

    def process_exception(self, request, exception, spider):
        return None

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

    def spider_closed(self, spider):
        self.bloomFilter.close()
        spider.logger.info('Spider closed: %s' % spider.name)


class DumpFilterSpiderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        s = cls()
        redis_host = crawler.spider.settings['REDIS_HOST']
        redis_port = crawler.spider.settings['REDIS_PORT']
        pwd = crawler.spider.settings['REDIS_PWD']
        s.bloomFilter = Client(host=redis_host, port=redis_port, password=pwd)
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s

    def process_spider_output(self, response, result, spider):
        for i in result:
            # 对于符合爬取条件的请求，加入判重
            if isinstance(i, scrapy.Item):
                if 'CUSTOM_FILTER_KEY' in response.request.meta and 'CUSTOM_FILTER_VAL' in response.request.meta:
                    key = response.request.meta['CUSTOM_FILTER_KEY']
                    val = response.request.meta['CUSTOM_FILTER_VAL']
                    if i['es_carriertype'] != 'news':
                        val = i['es_urlname']
                    if self.bloomFilter.bfAdd(key, val) <= 0:
                        spider.logger.debug('检测到重复URL，已过滤：%s' % val)
                        continue
            yield i

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

    def spider_closed(self, spider):
        self.bloomFilter.close()
        spider.logger.info('Spider closed: %s' % spider.name)


class SimhashFilterSpiderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        s = cls()
        redis_host = crawler.spider.settings['REDIS_HOST']
        redis_port = crawler.spider.settings['REDIS_PORT']
        pwd = crawler.spider.settings['REDIS_PWD']
        s.bloomFilter = Client(host=redis_host, port=redis_port, password=pwd)
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s

    def process_spider_input(self, response, spider):
        return None

    def process_spider_output(self, response, result, spider):
        for i in result:
            # 内容判重
            if isinstance(i, scrapy.Item):
                hash = Simhash(i['es_urlcontent']).value
                if self.bloomFilter.bfAdd('SimHash_Filter:' + i['es_srcname'], hash) <= 0:
                    spider.logger.info('检测到相同内容，已进行 SimHash 过滤：%s' % i['es_urlname'])
                    spider.logger.info('title:' + i['es_urltitle'])
                    continue
            yield i

    def get_features(self, s):
        width = 3
        s = s.lower()
        s = re.sub(r'[^\w]+', '', s)
        return [s[i:i + width] for i in range(max(len(s) - width + 1, 1))]

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

    def spider_closed(self, spider):
        self.bloomFilter.close()
        spider.logger.info('Spider closed: %s' % spider.name)
