import json
import logging
import time

import redis
import scrapy
from lxml import etree
from lxml.html import tostring
from scrapy.utils.project import get_project_settings

from MediaSpiders.items import MediaspidersItem
from MediaSpiders.utils.date_utils import get_time_stamp
from MediaSpiders.utils.http_utils import filter_html_tags, http_post


class WechatArticleSpider(scrapy.Spider):
    name = 'wechat_article'
    custom_settings = {
        'PROTO_MODULE_PATH': 'MediaSpiders.proto.Es_pb2',
        'PROTO_CLASS_NAME': 'EsSets',
        'PROTO_FIELD_NAME': 'Es',
        'PROTO_SAVE_FILE_NAME': 'public_info_data_',
        'IMAGES_STORE': r'/usr/local/temp_image/wechat',
        'IMAGES_URLS_FIELD': 'es_urlimage',
        'IMAGES_RESULT_FIELD': 'es_urlimage',
        'FILES_STORE': r'/usr/local/videos',
        'FILES_RESULT_FIELD': 'es_video',
        'ZIP_FILE_NAME': 'image_data_publicinfo_',
        'FILE_ZIP_FILE_NAME': 'image_data_plane_',
        'DOWNLOAD_DELAY': 1.5,
        'BATCH_SAVE_SIZE': 3,
        'ITEM_PIPELINES': {
            'MediaSpiders.pipelines.ProtobufSavePipeline': 300,
            'scrapy.pipelines.files.FilesPipeline': 1,
            'scrapy.pipelines.images.ImagesPipeline': 2,
        },
        'DOWNLOADER_MIDDLEWARES': {
            'MediaSpiders.middlewares.DumpFilterDownloaderMiddleware': 543,
            'MediaSpiders.middlewares.UserAgentDownloaderMiddleware': 545,
        },
        'SPIDER_MIDDLEWARES': {
            # 'MediaSpiders.middlewares.DumpFilterSpiderMiddleware': 543,
            # 'MediaSpiders.middlewares.SimhashFilterSpiderMiddleware': 545,
        }
    }
    settings = get_project_settings()

    article_batch_number = 50
    article_count = 0
    end_flag = False

    def __init__(self, params=None, *args, **kwargs):
        super(WechatArticleSpider, self).__init__(*args, **kwargs)
        self.redis_client = None
        if params:
            json_params = json.loads(params)
            if 'job_id' in json_params:
                self.job_id = json_params['job_id']

    def start_requests(self):
        self.redis_client = redis.Redis(host=self.settings['REDIS_HOST'], port=self.settings['REDIS_PORT'],
                                        password=self.settings['REDIS_PWD'])
        self.simhash_filter_key = self.settings['WECHAT_SIMHASH_FILTER_KEY']
        account_query_api = self.settings['SOCIAL_USER_QUERY_ALL_API']
        account_query_api = account_query_api.format(sortBy="id", shuffleResult="true")
        post_data = {
            'userType': self.settings['WECHAT_USER_TYPE'],
            'userFlag': 0
        }
        account_rsp = json.loads(
            http_post(account_query_api, json.dumps(post_data), headers={"Content-Type": "application/json"}).text)
        if account_rsp['code'] != 200 or len(account_rsp['content']) == 0:
            logging.error(f'微信账号信息接口调用失败：{account_query_api}')
            return
        accounts_info = account_rsp['content']
        for account in accounts_info:
            account_id = account['id']
            target_key = f'{self.settings["WECHAT_LINKS_KEY"]}:{account_id}'
            all_links = self.redis_client.smembers(target_key)
            if len(all_links) == 0 or all_links is None:
                logging.info(f'账号为：[{account["userName"]}]的公众号链接为空！')
                continue
            api_params = {"id": account['keywords']}
            parser_rsp = http_post(self.settings['RULES_PARSER_QUERY_API'], json.dumps(api_params),
                                   headers={"Content-Type": "application/json"})
            parser_json = json.loads(parser_rsp.text)
            parser_info = {}
            if parser_json['code'] == 200:
                parser_info = parser_json['content'][0]
            for link in all_links:
                yield scrapy.Request(url=link.decode(), callback=self.parse, meta={
                    'CUSTOM_FILTER_KEY': self.settings['WECHAT_FILTER_KEY'],
                    'CUSTOM_FILTER_VAL': link.decode(),
                    'PARSER_INFO': parser_info
                })
                self.redis_client.srem(target_key, link.decode())
                self.article_count += 1
                if self.article_count >= self.article_batch_number:
                    self.end_flag = True
                    break
            if self.end_flag:
                logging.info(f"本次获取文章链接数已达到设定的最大值{self.article_batch_number}，不再解析新的文章链接。"
                             f"未获取的链接将在下一次采集引擎运行时进行获取和采集。")
                break

    def parse(self, response, **kwargs):
        parser_info = response.request.meta['PARSER_INFO']
        rule_info = json.loads(parser_info['itemPropMap'])
        title_rule = rule_info['es_urltitle']
        content_rule = rule_info['es_urlcontent']
        image_rule = rule_info['es_urlimage']
        authors_rule = rule_info['es_authors']
        html_content = response.text
        selector = etree.HTML(html_content)
        content_tag_list = selector.xpath(content_rule)
        page_content = ''
        for tag in content_tag_list:
            content_html = tostring(tag, encoding="utf-8").decode("utf-8")
            page_content += filter_html_tags(content_html) + "<br>"
        if page_content.startswith("<br>"):
            page_content = page_content[4:]
        if page_content.endswith("<br>"):
            page_content = page_content[:-4]
        page_content = page_content.replace("<br>", "<br>\r\n")
        title = ''.join(selector.xpath(title_rule)).strip()
        image_urls = selector.xpath(image_rule)
        author = ''.join(selector.xpath(authors_rule)).strip()
        page_url = ''.join(selector.xpath("//meta[@property='og:url']/@content")).strip()
        time_index = html_content.find("var createTime")
        if time_index > 0:
            time_index_end = time_index + 36
            publish_time = html_content[time_index: time_index_end]
            publish_time = publish_time[18:-2]
            page_item = MediaspidersItem()
            page_item['es_urltitle'] = title
            page_item['es_urlcontent'] = page_content
            page_item['es_authors'] = author
            page_item['es_urlname'] = page_url
            page_item['es_urltime'] = get_time_stamp(publish_time)
            page_item['es_lasttime'] = page_item['es_urltime']
            page_item['es_urlimage'] = image_urls
            page_item['es_attachment'] = []
            page_item['es_tags'] = ''
            page_item['es_sitename'] = '微信公众号'
            page_item['es_catalog'] = 'news'
            page_item['es_srcname'] = 'http://mp.weixin.qq.com/'
            page_item['es_loadtime'] = str(int(time.time() * 1000))
            page_item['es_carriertype'] = 'wechat'
            page_item['es_doclength'] = len(page_content)
            page_item['image_min_width'] = rule_info['image_min_width']
            page_item['image_min_height'] = rule_info['image_min_height']
            if rule_info.get('custom_image_filter'):
                page_item['custom_image_filter'] = rule_info['custom_image_filter'].split(',')
            yield page_item
