# -*- coding: utf-8 -*-

import json
import logging
import random
import re
import time

import redis
import scrapy
from redisbloom.client import Client
from scrapy import signals
from scrapy.exceptions import IgnoreRequest
from scrapy.utils.project import get_project_settings
from simhash import Simhash

from WebsiteSpider.scrapy_selenium import SeleniumRequest
from WebsiteSpider.utils.http_utils import http_get
from WebsiteSpider.utils.http_utils import http_post
from WebsiteSpider.utils.string_utils import check_keywords

project_settings = get_project_settings()


# InitDownloaderMiddleware -- 获取采集规则，并根据星级或列表分类返回
def get_crawl_rule(spider, star, website_id):
    crawl_rule_info = []
    crawl_rule_query_api = project_settings['CRAWL_RULE_QUERY_API']
    query_data_body = {'status': 0} if website_id is None else {'id': website_id}
    # 星级站点，添加weight参数，查询目标星级的采集规则
    if star is not None:
        query_data_body['weight'] = star
    rsp = http_post(crawl_rule_query_api, json.dumps(query_data_body), headers={"Content-Type": "application/json"})
    rsp_body = json.loads(rsp.text)
    try:
        if len(rsp_body['content']) == 0:
            spider.logger.info('站点规则列表为空！')
            return False, crawl_rule_info
    except:
        spider.logger.info('获取采集站点规则库失败！')
        return False, crawl_rule_info
    if website_id is None:
        # 星级：0，列表：1
        crawl_method = '1' if star is None else '0'
        crawl_rule_info = [rule for rule in rsp_body['content'] if rule['parserConfig']['crawlMethod'] == crawl_method]
    else:
        crawl_rule_info = rsp_body['content']
    if len(crawl_rule_info) == 0:
        spider.logger.info('站点规则列表为空！')
        return False, crawl_rule_info
    return True, crawl_rule_info


# InitDownloaderMiddleware -- 将接口中参数与采集规则需要的参数进行对齐
def handle_crawl_rule_param(rule_info):
    try:
        url = rule_info['startUrl']
        parser_config = rule_info['parserConfig']
        filter_config = rule_info['filterConfig']
        crawl_param = {
            'detailPageReg': parser_config['detailPageReg'],
            'fieldMappings': json.loads(parser_config['itemPropMap'].replace("\\", "\\\\")),
            'isDynamicSite': parser_config['useSelenium'] == '1',
            'proxyMethod': parser_config['useProxy'],
            'keywordFilter': {
                'url_in': filter_config['urlIn'],
                'url_ex': filter_config['urlEx'],
                'title_in': filter_config['titleIn'],
                'title_ex': filter_config['titleEx'],
                'content_in': filter_config['contentIn'],
                'content_ex': filter_config['contentEx'],
                'global_in': json.loads(filter_config['globalIn']),
                'global_ex': json.loads(filter_config['globalEx']),
            },
            'region': rule_info['region'],
            'url': url,
            'systemFlag': rule_info['project'],
            'resourceType': rule_info.get('resourceType', ''),
            'storageOption': json.loads(rule_info['storageOption'])
        }
        # 参数名相同的统一赋值
        parser_key_to_add = ['allowDomain', 'cron', 'depth', 'timeZone', 'crawlMethod']
        crawl_param.update({key: parser_config[key] for key in parser_key_to_add if key in parser_config})
        rule_key_to_add = ['id', 'siteName', 'siteType', 'carrierType', 'weight', 'resourceType']
        crawl_param.update({key: rule_info[key] for key in rule_key_to_add if key in rule_info})
        # logging.info(crawl_param[0])
        return crawl_param, url
    except:
        return None, None


class InitDownloaderMiddleware(object):
    def __init__(self):
        self.redis_client = None
        self.crawl_rule = {}

    @classmethod
    def from_crawler(cls, crawler):
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s

    def process_request(self, request, spider):
        parsed_link = request.url
        if 'parse_rule' not in request.meta:
            if parsed_link in self.crawl_rule:
                parse_rule = self.crawl_rule[parsed_link]
                parse_rule['current_depth'] = 1
                parse_rule['start_url'] = parsed_link
                request.meta['parse_rule'] = parse_rule
                if request.meta['parse_rule']['isDynamicSite']:
                    logging.debug("发送 Selenium 请求：" + request.url)
                    return SeleniumRequest(url=request.url, meta=request.meta)
        return None

    def spider_opened(self, spider):
        spider.logger.info('InitDownloaderMiddleware Enabled')
        CONCURRENT_REQUESTS = spider.settings['CONCURRENT_REQUESTS']
        spider.logger.info(f'本次采集任务的并发线程数为：{CONCURRENT_REQUESTS}')
        self.redis_client = redis.Redis(host=spider.settings['REDIS_HOST'], port=int(spider.settings['REDIS_PORT']),
                                        password=spider.settings['REDIS_PWD'])
        # 更新采集规则
        spider_name = spider.name
        # 单站点采集逻辑
        if not spider_name.startswith('WebSite_star'):
            website_id = spider_name.split("_")[-1]
            flag, crawl_rule_info = get_crawl_rule(spider, None, website_id)
            if not flag:
                return
            for rule_info in crawl_rule_info:
                # 将接口中参数与采集规则需要的参数进行对齐
                crawl_param, url = handle_crawl_rule_param(rule_info)
                if crawl_param is None or url is None:
                    continue
                self.crawl_rule[url] = crawl_param
            spider.logger.info(f"单站点采集规则更新")
            return
        star = spider_name.split('_')[-1]
        request_queue_name = f'WebSite_star_{star}:requests'
        start_urls_queue_name = f'WebSite_star_{star}:start_urls'
        flag, crawl_rule_info = get_crawl_rule(spider, star, None)
        if not flag:
            return
        dist_flag = True
        if self.redis_client.exists(request_queue_name) or self.redis_client.exists(start_urls_queue_name):
            spider.logger.debug(f'由于星级为{star}的站点没有完成采集，暂时不分发当前星级站点')
            dist_flag = False
        dupe_filter_queue_name = f'WebSite_star_{star}:dupefilter'
        if dist_flag:
            self.redis_client.delete(dupe_filter_queue_name)
        # 获取采集规则，并根据星级或列表分类返回
        count = 0
        update_start_time = time.perf_counter()
        # 对站点进行处理，形成符合采集引擎要求的形式，然后按照权重分组
        for rule_info in crawl_rule_info:
            # 将接口中参数与采集规则需要的参数进行对齐
            crawl_param, url = handle_crawl_rule_param(rule_info)
            if crawl_param is None or url is None:
                continue
            self.crawl_rule[url] = crawl_param
            if dist_flag:
                self.redis_client.zadd(f'WebSite_star_{star}:start_urls', {url: star})
            count += 1
        update_duration = time.perf_counter() - update_start_time
        spider.logger.info(f"共{count}个站点，更新耗时：{update_duration:.2f}秒")

    def spider_closed(self, spider):
        if self.redis_client is not None:
            if re.match(r"WebSite_\w+(\.\w+)+_\w{8}\-\w{4}\-\w{4}\-\w{4}\-\w{12}", spider.name):
                self.redis_client.delete('{}:dupefilter'.format(spider.name))
                spider.logger.info("采集结束，删除单站点判重队列 {}".format('{}:dupefilter'.format(spider.name)))
        self.redis_client.close()
        spider.logger.info('[InitDownloaderMiddleware] 爬虫 {} 关闭，同时关闭 Redis client'.format(spider.name))


class InitReportListDownloaderMiddleware(object):
    def __init__(self):
        self.redis_client = None
        self.crawl_report_rule = {}

    @classmethod
    def from_crawler(cls, crawler):
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s

    def process_request(self, request, spider):
        parsed_link = request.url
        if 'parse_rule' not in request.meta:
            if parsed_link in self.crawl_report_rule:
                parse_rule = self.crawl_report_rule[parsed_link]
            else:
                return None
            parse_rule['current_depth'] = 1
            parse_rule['start_url'] = parsed_link
            request.meta['parse_rule'] = parse_rule
            if request.meta['parse_rule']['isDynamicSite']:
                return SeleniumRequest(url=request.url, meta=request.meta)
        return None

    def spider_opened(self, spider):
        spider.logger.info('InitReportListDownloaderMiddleware Enabled')
        self.redis_client = redis.Redis(host=spider.settings['REDIS_HOST'], port=int(spider.settings['REDIS_PORT']),
                                        password=spider.settings['REDIS_PWD'])
        # 获取采集规则，并根据星级或列表分类返回
        flag, crawl_rule_info = get_crawl_rule(spider, None, None)
        if not flag:
            return
        dist_flag = True
        if self.redis_client.exists('Website_report_list:requests') or self.redis_client.exists(
                'Website_report_list:start_urls'):
            spider.logger.debug(f'由于列表站点没有完成采集，暂时不分发当前列表站点')
            dist_flag = False
        if dist_flag:
            self.redis_client.delete('Website_report_list:dupefilter')
        count = 0
        update_start_time = time.perf_counter()
        # 对站点进行处理，形成符合采集引擎要求的形式，然后按照权重分组
        for rule_info in crawl_rule_info:
            # 将接口中参数与采集规则需要的参数进行对齐
            crawl_param, url = handle_crawl_rule_param(rule_info)
            if crawl_param is None or url is None:
                continue
            # 参数对齐完成后，存到内存变量中，等待程序调用
            self.crawl_report_rule[url] = crawl_param
            # 对于每个站点解析模板，若某个字段无自定义规则，则将采用默认模板中规则
            if dist_flag:
                self.redis_client.zadd('Website_report_list:start_urls', {url: rule_info['weight']})
            count += 1
        update_duration = time.perf_counter() - update_start_time
        spider.logger.info(f"共{count}个站点，更新耗时：{update_duration:.2f}秒")

    def spider_closed(self, spider):
        self.redis_client.close()
        spider.logger.info('spider closed')


class AutoProxyDownloaderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.
        if 'parse_rule' not in request.meta:
            return None
        if request.meta['parse_rule']['proxyMethod'] == '1':
            # 使用动态代理(ip池)
            if self.request_count == 0 or self.request_count >= int(spider.settings['PER_BATCH_IP_USE_TIMES']):
                self.request_count = 0
                self.proxy_ip_list = []
                proxy_service = spider.settings['PROXY_SERVICE']
                # 调用自建Ip池api
                self.proxy_ip_list = json.loads(http_get(proxy_service + '/get_all').text)
                while len(self.proxy_ip_list) == 0:
                    self.proxy_ip_list = json.loads(http_get(proxy_service + '/get_all').text)
                    time.sleep(1)
            proxy = random.choice(self.proxy_ip_list)
            self.request_count += 1
            if 'splash' in request.meta:
                request.meta['splash']['args']['proxy'] = 'http://%s' % proxy['proxy']
            else:
                request.meta['proxy'] = 'http://%s' % proxy['proxy']
        return None

    def process_exception(self, request, exception, spider):
        # if isinstance(exception,ConnectionRefusedError):
        #     proxy_service = spider.settings['PROXY_SERVICE']
        #     http_get(proxy_service + 'delete?proxy=' + request.meta['proxy'].split('/')[-1])
        #     return request
        return None

    def spider_opened(self, spider):
        self.request_count = 0
        spider.logger.info('Spider opened: %s' % spider.name)


class UserAgentDownloaderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        s.uas = crawler.spider.settings['CUSTOM_USER_AGENT']
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        UA = random.choice(self.uas)
        if request.headers is None:
            request.headers = {}
        request.headers['User-Agent'] = UA
        return None

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class DeepLimitDownloaderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        # s.depth = crawler.spider.settings['CRAWL_DEPTH']
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # 爬取深度限制
        if 'parse_rule' not in request.meta:
            return None
        site_depth = int(request.meta['parse_rule']['depth'])
        if site_depth <= 0:
            site_depth = 2
        if request.meta['parse_rule']['current_depth'] > site_depth:
            raise IgnoreRequest()
        return None

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class DumpFilterDownloaderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        redis_host = crawler.spider.settings['REDIS_HOST']
        redis_port = crawler.spider.settings['REDIS_PORT']
        pwd = crawler.spider.settings['REDIS_PWD']
        s.bloomFilter = Client(host=redis_host, port=redis_port, password=pwd)
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s

    def process_request(self, request, spider):
        # 判重
        if 'CUSTOM_FILTER_KEY' in request.meta and 'CUSTOM_FILTER_VAL' in request.meta:
            key = request.meta['CUSTOM_FILTER_KEY']
            val = request.meta['CUSTOM_FILTER_VAL']
            if self.bloomFilter.bfExists(key, val) == 1:
                spider.logger.debug('检测到重复 URL，已过滤：%s' % val)
                raise IgnoreRequest()
        return None

    def process_exception(self, request, exception, spider):
        return None

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

    def spider_closed(self, spider):
        self.bloomFilter.close()
        spider.logger.info('Spider closed: %s' % spider.name)


class DumpFilterSpiderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        s = cls()
        redis_host = crawler.spider.settings['REDIS_HOST']
        redis_port = crawler.spider.settings['REDIS_PORT']
        pwd = crawler.spider.settings['REDIS_PWD']
        s.bloomFilter = Client(host=redis_host, port=redis_port, password=pwd)
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s

    def process_spider_output(self, response, result, spider):
        for i in result:
            # 对于符合爬取条件的请求，加入判重
            if isinstance(i, scrapy.Item):
                if 'CUSTOM_FILTER_KEY' in response.request.meta and 'CUSTOM_FILTER_VAL' in response.request.meta:
                    key = response.request.meta['CUSTOM_FILTER_KEY']
                    val = response.request.meta['CUSTOM_FILTER_VAL']
                    if i['es_carriertype'] != 'news':
                        val = i['es_urlname']
                    if self.bloomFilter.bfAdd(key, val) <= 0:
                        spider.logger.debug('检测到重复URL，已过滤：%s' % val)
                        continue
            yield i

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

    def spider_closed(self, spider):
        self.bloomFilter.close()
        spider.logger.info('Spider closed: %s' % spider.name)


class SimhashFilterSpiderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        s = cls()
        redis_host = crawler.spider.settings['REDIS_HOST']
        redis_port = crawler.spider.settings['REDIS_PORT']
        pwd = crawler.spider.settings['REDIS_PWD']
        s.bloomFilter = Client(host=redis_host, port=redis_port, password=pwd)
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s

    def process_spider_input(self, response, spider):
        return None

    def process_spider_output(self, response, result, spider):
        for i in result:
            # 内容判重
            if isinstance(i, scrapy.Item):
                hash = Simhash(i['es_urlcontent']).value
                if self.bloomFilter.bfAdd('SimHash_Filter:' + i['es_srcname'], hash) <= 0:
                    spider.logger.info('检测到相同内容，已进行 SimHash 过滤：%s' % i['es_urlname'])
                    spider.logger.info('title:' + i['es_urltitle'])
                    continue
            yield i

    def get_features(self, s):
        width = 3
        s = s.lower()
        s = re.sub(r'[^\w]+', '', s)
        return [s[i:i + width] for i in range(max(len(s) - width + 1, 1))]

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

    def spider_closed(self, spider):
        self.bloomFilter.close()
        spider.logger.info('Spider closed: %s' % spider.name)


# 该过滤器在请求后生效，判断文章标题和内容，若符合过滤规则,则该请求结果不会保存
class KeywordFilterSpiderMiddleware(object):
    def __init__(self):
        self.bloomFilter = None
        self.query_api = ''
        self.word_by_category = {}

    def query_words(self, category_list, lang_list=None):
        result_list = []
        for cate in category_list:
            if cate not in self.word_by_category:
                self.word_by_category[cate] = []
                data_body = {
                    'category': [cate]
                }
                if lang_list:
                    data_body['lang'] = lang_list
                query_result = json.loads(http_post(self.query_api, json.dumps(data_body),
                                                    {'Content-Type': 'application/json'}).content.decode())
                if query_result['code'] == 200:
                    for word_item in query_result['content']:
                        if word_item['delFlag'] == 0:
                            self.word_by_category[cate].append(word_item['word'])
                    logging.info("查询到分类码 {} 的关键词 {} 个".format(cate, len(self.word_by_category[cate])))
                else:
                    logging.error("关键词查询接口返回值错误：" + query_result['message'])
            result_list += self.word_by_category[cate]
        return result_list

    @classmethod
    def from_crawler(cls, crawler):
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s

    def process_spider_output(self, response, result, spider):
        for i in result:
            if isinstance(i, scrapy.Item):
                title_in_words = []
                title_ex_words = []
                content_in_words = []
                content_ex_words = []
                parse_rule = response.request.meta['parse_rule']
                filter_rule = parse_rule['keywordFilter']
                if 'title_in' in filter_rule and len(filter_rule['title_in']) > 0:
                    title_in_words = filter_rule['title_in'].split(';')
                if 'title_ex' in filter_rule and len(filter_rule['title_ex']) > 0:
                    title_ex_words = filter_rule['title_ex'].split(';')
                if 'content_in' in filter_rule and len(filter_rule['content_in']) > 0:
                    content_in_words = filter_rule['content_in'].split(';')
                if 'content_ex' in filter_rule and len(filter_rule['content_ex']) > 0:
                    content_ex_words = filter_rule['content_ex'].split(';')
                if 'global_in' in filter_rule:
                    category = filter_rule['global_in']
                    if len(category) > 0:
                        word_bank_in = self.query_words(category_list=category)
                        content_in_words += word_bank_in
                        title_in_words += word_bank_in
                if 'global_ex' in filter_rule:
                    category = filter_rule['global_ex']
                    if len(category) > 0:
                        word_bank_ex = self.query_words(category_list=category)
                        content_ex_words += word_bank_ex
                        title_ex_words += word_bank_ex

                title_in_words = set(title_in_words)
                title_ex_words = set(title_ex_words)
                content_in_words = set(content_in_words)
                content_ex_words = set(content_ex_words)

                title_checked = check_keywords(content=i['es_urltitle'], include=title_in_words, exclude=title_ex_words)
                content_checked = check_keywords(content=i['es_urlcontent'], include=content_in_words,
                                                 exclude=content_ex_words)
                yield_flag = title_checked and content_checked
                if yield_flag:
                    # 将实际的标题（不含板块名称）赋值给es_urltitle
                    if 'es_reportinfo' in i and len(i['es_reportinfo']) > 0:
                        spider.logger.info('原始文章标题为：' + i['es_urltitle'])
                        i['es_urltitle'] = i['es_reportinfo']
                        spider.logger.info('文章标题替换为：' + i['es_urltitle'])
                        i['es_reportinfo'] = ''
                else:
                    continue
            yield i

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)
        redis_host = spider.settings['REDIS_HOST']
        redis_port = spider.settings['REDIS_PORT']
        self.query_api = spider.settings['WORD_BANK_QUERY_API']
        pwd = spider.settings['REDIS_PWD']
        self.bloomFilter = Client(host=redis_host, port=redis_port, password=pwd)

    def spider_closed(self, spider):
        self.bloomFilter.close()
        spider.logger.info('Spider closed: %s' % spider.name)


# 该过滤器在发送request前生效，判断请求的URL，若符合过滤规则，将不再发送请求
class KeywordFilterDownloaderMiddleware(object):

    @classmethod
    def from_crawler(cls, crawler):
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        if 'parse_rule' not in request.meta:
            return None
        if request.url == request.meta['parse_rule']['url']:
            return None
        url_in_key_words = []
        url_ex_key_words = []
        parse_rule = request.meta['parse_rule']
        filter_rule = parse_rule['keywordFilter']
        if 'url_in' in filter_rule and len(filter_rule['url_in']) > 0:
            url_in_key_words = filter_rule['url_in'].split(';')
        if 'url_ex' in filter_rule and len(filter_rule['url_ex']) > 0:
            url_ex_key_words = filter_rule['url_ex'].split(';')
        url_checked = check_keywords(content=request.url, include=url_in_key_words, exclude=url_ex_key_words)
        if not url_checked:
            raise IgnoreRequest()
        else:
            return None

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)
