# -*- coding: utf-8 -*-
import json
import time

import scrapy
from redisbloom.client import Client

from MediaSpiders.items import MediaspidersItem
from MediaSpiders.utils.http_utils import http_post
from MediaSpiders.utils.string_utils import find_text
from MediaSpiders.utils.time_utils import get_time_stamp, get_current_timestamp


class WeiboSpider(scrapy.Spider):
    name = 'WeiboUserSpider'
    allowed_domains = ['m.weibo.cn', 'sinaimg.cn', 'video.weibocdn.com']
    custom_settings = {
        'PROTO_MODULE_PATH': 'MediaSpiders.proto.Es_pb2',
        'PROTO_CLASS_NAME': 'EsSets',
        'PROTO_FIELD_NAME': 'Es',
        'PROTO_SAVE_FILE_NAME': 'public_info_data_',
        'IMAGES_STORE': r'/usr/local/temp_image/publicinfo',
        'IMAGES_RESULT_FIELD': 'es_urlimage',
        'FILES_STORE': r'/usr/local/videos',
        'FILES_RESULT_FIELD': 'es_video',
        'ZIP_FILE_NAME': 'image_data_publicinfo_',
        'FILE_ZIP_FILE_NAME': 'image_data_plane_',
        'DOWNLOAD_DELAY': 3,
        'BATCH_SAVE_SIZE': 20,
        'ITEM_PIPELINES': {
            'scrapy.pipelines.images.ImagesPipeline': 2,
            'scrapy.pipelines.files.FilesPipeline': 1,
            'MediaSpiders.pipelines.ProtobufSavePipeline': 300
        }
    }

    def __init__(self, params=None, *args, **kwargs):
        super(WeiboSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        self.crawl_video = False
        if 'crawl_video' in json_params and json_params['paramsObj'] == '1':
            self.crawl_video = True
        if 'totalNum' in json_params:
            self.total_num = int(json_params['totalNum'])
        else:
            self.total_num = 20
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def start_requests(self):
        self.bloom_filter = Client(host=self.settings['REDIS_HOST'], port=self.settings['REDIS_PORT'],
                                   password=self.settings['REDIS_PWD'])
        account_query_api = self.settings['SOCIAL_USER_QUERY_ALL_API']
        account_query_api = account_query_api.format(sortBy="id", shuffleResult="true")
        post_data = {
            'userType': self.settings['WEIBO_USER_TYPE'],
            'userFlag': 0
        }
        account_rsp = json.loads(
            http_post(account_query_api, json.dumps(post_data), headers={"Content-Type": "application/json"}).text)
        self.simhash_filter_key = self.settings['WEIBO_SIMHASH_FILTER_KEY']
        all_user_info = []
        if account_rsp['code'] == 200:
            all_user_info = account_rsp['content']
        for user_info in all_user_info:
            uid = user_info['userUid']
            if uid[:6] != '107603':
                uid = f'107603{uid}'
            yield scrapy.Request('https://m.weibo.cn/api/container/getIndex?containerid=%s' % uid,
                                 callback=self.parse, meta={'currentCount': 0, 'uid': uid})

    def parse(self, response):
        rsp = json.loads(response.text)
        if 'ok' not in rsp or rsp['ok'] != 1 or 'data' not in rsp or 'cards' not in rsp['data']:
            return
        current_count = response.request.meta['currentCount']
        uid = response.request.meta['uid']
        cards = rsp['data']['cards']
        for card in cards:
            if self.total_num > 0 and current_count >= self.total_num:
                return
            if card['card_type'] != 9:
                continue
            current_count += 1
            # 判重
            if self.bloom_filter.bfAdd(self.settings['WEIBO_FILTER_KEY'], card['mblog']['id']) <= 0:
                continue
            mblog = card['mblog']
            item = MediaspidersItem()
            item['es_userid'] = str(uid)
            item['es_sid'] = mblog['id']
            item['es_hkey'] = mblog['id']
            item['es_urlname'] = 'https://m.weibo.cn/detail/%s' % mblog['id']
            item['es_sitename'] = '微博'
            item['es_lasttime'] = get_current_timestamp()
            item['es_loadtime'] = get_current_timestamp()
            item['es_urldate'] = get_time_stamp(mblog['created_at'])
            if item['es_urldate'] is None:
                item['es_urldate'] = str(int(time.time() * 1000))
            item['es_urltime'] = get_time_stamp(mblog['created_at'])
            if item['es_urltime'] is None:
                item['es_urltime'] = str(int(time.time() * 1000))
            item['es_srcname'] = '微博'
            item['es_authors'] = mblog['user']['screen_name']
            item['es_carriertype'] = 'media'
            item['es_commentcount'] = mblog['comments_count']
            item['es_forwardcount'] = mblog['reposts_count']
            item['es_likecount'] = mblog['attitudes_count']
            author_full_text = find_text(mblog['text'])
            if 'retweeted_status' in mblog:
                item['es_isrepost'] = 'yes'
                item['es_urltitle'] = author_full_text
                item['es_catalog1'] = author_full_text
                mblog = mblog['retweeted_status']
                self.logger.info("采集转发内容")
            item['es_content'] = find_text(mblog['text'])
            item['es_urlcontent'] = find_text(mblog['text'])
            if 'pics' in mblog:
                urlList = []
                # for img in mblog['pics']:
                #     urlList.append(img['url'])
                # 优先下载更清晰的图片
                for img in mblog['pics']:
                    if 'large' in img and 'url' in img['large'] and img['large']['url'] != '':
                        urlList.append(img['large']['url'])
                    else:
                        urlList.append(img['url'])
                item['image_urls'] = urlList
            if self.crawl_video:
                if 'page_info' in mblog and mblog['page_info']['type'] == 'video':
                    videoUrlList = []
                    video_urls = mblog['page_info']['urls']
                    if 'mp4_ld_mp4' in video_urls:
                        target_url = video_urls['mp4_ld_mp4']
                        # self.logger.info('获取到视频，地址为：%s' % target_url)
                        # if '?' in target_url:
                        #     target_url = target_url.split('?')[0]
                        videoUrlList.append(target_url)
                    item['file_urls'] = videoUrlList

            # 加入判重因子#
            response.request.meta['CUSTOM_FILTER_KEY'] = self.settings['WEIBO_FILTER_KEY']
            response.request.meta['CUSTOM_FILTER_VAL'] = item['es_sid']
            # 判断是否需要获取博文详细内容#
            retweetedContent = ''
            retweetedId = ''
            if 'retweeted_status' in mblog:
                retweetedContent = find_text(mblog['retweeted_status']['text'])
                retweetedId = mblog['retweeted_status']['id']
            originFlag, retweetedFlag = False, False
            if item['es_content'].find('全文') > 0:
                originFlag = True
            if retweetedContent.find('全文') > 0:
                retweetedFlag = True
            if not originFlag and not retweetedFlag:
                if len(retweetedContent) > 0:
                    item['es_content'] += r'\n转发内容：' + retweetedContent
                yield item
                continue
            if originFlag:
                yield scrapy.Request('https://m.weibo.cn/statuses/extend?id=%s' % mblog['id'],
                                     callback=self.blog_detail_parse1,
                                     meta={'item': item, 'retweetedFlag': retweetedFlag,
                                           'retweetedContent': retweetedContent, 'retweetedId': retweetedId})
                continue
            yield scrapy.Request('https://m.weibo.cn/statuses/extend?id=%s' % mblog['retweeted_status']['id'],
                                 callback=self.blog_detail_parse2, meta={'item': item})

        if 'since_id' in rsp['data']['cardlistInfo']:
            since_id = rsp['data']['cardlistInfo']['since_id']
            yield scrapy.Request(
                'https://m.weibo.cn/api/container/getIndex?luicode=10000011&containerid=%s&since_id=%s' % (
                    uid, since_id), meta={'currentCount': current_count, 'uid': uid})

    # 解析原文内容详情
    def blog_detail_parse1(self, response):
        item = response.request.meta['item']
        try:
            rsp = json.loads(response.text)
            if 'ok' in rsp and rsp['ok'] == 1 and 'data' in rsp:
                longText = find_text(rsp['data']['longTextContent'])
                item['es_content'] = longText
                item['es_urlcontent'] = longText

            retweetedFlag = response.request.meta['retweetedFlag']
            retweetedContent = response.request.meta['retweetedContent']
            retweetedId = response.request.meta['retweetedId']
            if retweetedFlag:
                yield scrapy.Request('https://m.weibo.cn/statuses/extend?id=%s' % retweetedId,
                                     callback=self.blog_detail_parse2, meta={'item': item})
            elif len(retweetedContent) > 0:
                item['es_content'] += r'\n转发内容：' + retweetedContent
                item['es_urlcontent'] = item['es_content']
        except:
            print('解析更多内容出错')
        yield item

    # 解析转发内容详情
    def blog_detail_parse2(self, response):
        item = response.request.meta['item']
        try:
            rsp = json.loads(response.text)
            if 'ok' in rsp and rsp['ok'] == 1 and 'data' in rsp:
                longText = find_text(rsp['data']['longTextContent'])
                item['es_content'] += r'\n转发内容：' + longText
                item['es_urlcontent'] = item['es_content']
        except:
            print('解析更多内容出错')
        yield item
