# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
import json
import logging
import random
import re
import time
from urllib import parse

import scrapy
from redisbloom.client import Client
from scrapy import signals
from scrapy.exceptions import IgnoreRequest
from simhash import Simhash

from MediaSpiders.utils.http_utils import http_get, http_post
from MediaSpiders.utils.string_utils import check_keywords


class DumpFilterDownloaderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        redisHost = crawler.spider.settings['REDIS_HOST']
        redisPort = crawler.spider.settings['REDIS_PORT']
        pwd = crawler.spider.settings['REDIS_PWD']
        s.bloomFilter = Client(host=redisHost, port=redisPort, password=pwd)
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # 判重
        if 'CUSTOM_FILTER_KEY' in request.meta and 'CUSTOM_FILTER_VAL' in request.meta:
            key = request.meta['CUSTOM_FILTER_KEY']
            val = request.meta['CUSTOM_FILTER_VAL']
            if self.bloomFilter.bfExists(key, val) == 1:
                raise IgnoreRequest()
        # elif self.bloomFilter.bfAdd('urls', request.url) == 0: 
        #     raise IgnoreRequest()   
        return None

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s. DumpFilterDownloaderMiddleware started' % (spider.name,))


class DumpFilterSpiderMiddleware(object):

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        redisHost = crawler.spider.settings['REDIS_HOST']
        redisPort = crawler.spider.settings['REDIS_PORT']
        pwd = crawler.spider.settings['REDIS_PWD']
        s.bloomFilter = Client(host=redisHost, port=redisPort, password=pwd)
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.
        # Must return an iterable of Request, dict or Item objects.
        for i in result:
            # 对于符合爬取条件的请求，加入判重
            if isinstance(i, scrapy.Item):
                if 'CUSTOM_FILTER_KEY' in response.request.meta and 'CUSTOM_FILTER_VAL' in response.request.meta:
                    key = response.request.meta['CUSTOM_FILTER_KEY']
                    val = response.request.meta['CUSTOM_FILTER_VAL']
                    if self.bloomFilter.bfAdd(key, val) <= 0:
                        spider.logger.debug('检测到重复的内容：%s' % val)
                        continue
            yield i

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s. DumpFilterSpiderMiddleware started' % spider.name)


class SimhashFilterSpiderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        s = cls()
        redis_host = crawler.spider.settings['REDIS_HOST']
        redis_port = crawler.spider.settings['REDIS_PORT']
        pwd = crawler.spider.settings['REDIS_PWD']
        s.bloomFilter = Client(host=redis_host, port=redis_port, password=pwd)
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s

    def process_spider_input(self, response, spider):
        return None

    def process_spider_output(self, response, result, spider):
        for i in result:
            # 内容判重
            if isinstance(i, scrapy.Item):
                hash = Simhash(i['es_urlcontent']).value
                if self.bloomFilter.bfAdd(spider.simhash_filter_key, hash) <= 0:
                    spider.logger.info('检测到相同内容，已进行SimHash过滤：%s' % i['es_urlname'])
                    spider.logger.info('content: ' + i['es_urlcontent'][:50])
                    continue
            yield i

    def get_features(self, s):
        width = 3
        s = s.lower()
        s = re.sub(r'[^\w]+', '', s)
        return [s[i:i + width] for i in range(max(len(s) - width + 1, 1))]

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

    def spider_closed(self, spider):
        self.bloomFilter.close()
        spider.logger.info('Spider closed: %s' % spider.name)


class MyProxyDownloaderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.
        if self.request_count == 0 or self.request_count >= int(spider.settings['PER_BATCH_IP_USE_TIMES']):
            self.request_count = 0
            self.proxyIpList = []
            proxy_service = spider.settings['PROXY_SERVICE']
            # 调用自建Ip池api
            self.proxyIpList = json.loads(http_get(proxy_service + 'get_all').text)
            while len(self.proxyIpList) == 0:
                self.proxyIpList = json.loads(http_get(proxy_service + 'get_all').text)
                time.sleep(1)

        proxy = random.choice(self.proxyIpList)
        request.meta['proxy'] = 'http://%s' % proxy['proxy']
        self.request_count += 1

        # 本地测试时，使用下面的if-else进行代理设置
        # if 'splash' in request.meta:
        #     request.meta['splash']['args']['proxy'] = 'http://192.168.199.243:10809'
        # else:
        #     request.meta['proxy'] = 'http://%s'%'192.168.199.243:10809'
        return None

    def process_exception(self, request, exception, spider):
        # if isinstance(exception,ConnectionRefusedError):
        #     proxy_service = spider.settings['PROXY_SERVICE']
        #     http_get(proxy_service + 'delete?proxy=' + request.meta['proxy'].split('/')[-1])
        #     return request
        return None

    def spider_opened(self, spider):
        self.request_count = 0
        spider.logger.info('Spider opened: %s. MyProxyDownloaderMiddleware started' % spider.name)


class UserAgentDownloaderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        s.uas = crawler.spider.settings['CUSTOM_USER_AGENT']
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        UA = random.choice(self.uas)
        if request.headers is None:
            request.headers = {}

        request.headers['User-Agent'] = UA
        request.headers['Host'] = parse.urlparse(request.url)[1]

        return None

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s. UserAgentDownloaderMiddleware started' % spider.name)


class KeywordFilterSpiderMiddleware(object):
    def __init__(self):
        self.bloomFilter = None
        self.query_api = ''
        self.word_by_category = {}
        self.global_exclude = []
        self.global_include = []

    def query_words(self, category_list, lang_list=None):
        result_list = []
        for cate in category_list:
            if cate not in self.word_by_category:
                self.word_by_category[cate] = []
                data_body = {
                    'category': [cate]
                }
                if lang_list:
                    data_body['lang'] = lang_list
                query_result = json.loads(http_post(self.query_api, json.dumps(data_body),
                                                    {'Content-Type': 'application/json'}).content.decode())
                if query_result['code'] == 200:
                    for word_item in query_result['content']:
                        if word_item['delFlag'] == 0:
                            self.word_by_category[cate].append(word_item['word'])
                    logging.info("查询到分类码 {} 的关键词 {} 个".format(cate, len(self.word_by_category[cate])))
                else:
                    logging.error("关键词查询接口返回值错误：" + query_result['message'])
            result_list += self.word_by_category[cate]
        return result_list

    @classmethod
    def from_crawler(cls, crawler):
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s

    def process_spider_output(self, response, result, spider):
        for i in result:
            if isinstance(i, scrapy.Item):
                yield_flag = check_keywords(i['es_urlcontent'], exclude=self.global_exclude)
                if not yield_flag:
                    continue
            yield i

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)
        redis_host = spider.settings['REDIS_HOST']
        redis_port = spider.settings['REDIS_PORT']
        pwd = spider.settings['REDIS_PWD']
        self.query_api = spider.settings['WORD_BANK_QUERY_API']
        self.bloomFilter = Client(host=redis_host, port=redis_port, password=pwd)
        self.global_exclude = self.query_words(category_list=["0"])

    def spider_closed(self, spider):
        self.bloomFilter.close()
        spider.logger.info('Spider closed: %s' % spider.name)
