# -*- coding: utf-8 -*-
import json
import logging as logger
import random
import re
import time
from urllib import parse

import redis
import scrapy
from scrapy_selenium import SeleniumRequest

from MediaSpiders.items import MediaspidersItem
from MediaSpiders.utils.http_utils import http_post
from MediaSpiders.utils.time_utils import get_time_stamp, get_current_timestamp


class TwitterSpider(scrapy.Spider):
    name = 'TwitterUserSpider'
    custom_settings = {
        'PROTO_MODULE_PATH': 'MediaSpiders.proto.Es_pb2',
        'PROTO_CLASS_NAME': 'EsSets',
        'PROTO_FIELD_NAME': 'Es',
        'PROTO_SAVE_FILE_NAME': 'public_info_data_',
        'IMAGES_STORE': r'/usr/local/temp_image/twitter',
        'IMAGES_RESULT_FIELD': 'es_urlimage',
        'FILES_STORE': r'/usr/local/videos',
        'FILES_RESULT_FIELD': 'es_video',
        'ZIP_FILE_NAME': 'image_data_publicinfo_',
        'FILE_ZIP_FILE_NAME': 'image_data_plane_',
        'ITEM_PIPELINES': {
            'scrapy.pipelines.images.ImagesPipeline': 2,
            'scrapy.pipelines.files.FilesPipeline': 1,
            'MediaSpiders.pipelines.ProtobufSavePipeline': 300,
        },
        'SPIDER_MIDDLEWARES': {
            'MediaSpiders.middlewares.DumpFilterSpiderMiddleware': 543,
            # 'MediaSpiders.middlewares.KeywordFilterSpiderMiddleware': 544,
            # 'MediaSpiders.middlewares.SimhashFilterSpiderMiddleware': 545,
            # 'scrapy_splash.SplashDeduplicateArgsMiddleware': 700,
        }

    }

    def __init__(self, params=None, *args, **kwargs):
        super(TwitterSpider, self).__init__(*args, **kwargs)
        self.total_num = 100
        self.authorization = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'
        if params:
            json_params = json.loads(params)
            if 'totalNum' in json_params:
                self.total_num = int(json_params['totalNum'])
            if 'authorization' in json_params:
                self.authorization = json_params['authorization']
            if 'job_id' in json_params:
                self.job_id = json_params['job_id']

    def start_requests(self):
        yield SeleniumRequest(url='https://www.google.com/', callback=self.login_twitter)

    def login_twitter(self, response):
        self.redis_client = redis.Redis(host=self.settings['REDIS_HOST'], port=self.settings['REDIS_PORT'],
                                        password=self.settings['REDIS_PWD'])
        self.simhash_filter_key = self.settings['TWITTER_SIMHASH_FILTER_KEY']
        logger.info("login twitter")
        driver = response.request.meta['driver']
        driver.maximize_window()
        driver.get('https://twitter.com/i/flow/login')
        time.sleep(5)
        # 获取采集登录账号并登录
        login_users = self.redis_client.smembers('MediaSpiders:Twitter_login_accounts')
        user_list = []
        for u in login_users:
            user_list.append(json.loads(u.decode()))
        login_user = random.choice(user_list)
        logger.info(f"login as user {login_user['uid']}")
        driver.find_element_by_xpath("//input").send_keys(login_user['uid'])
        try:
            next_button = driver.find_element_by_xpath("//div[@role='button'][2]")
            next_button.click()
        except Exception:
            logger.info("点击“下一步”的button元素")
            next_button = driver.find_element_by_xpath("//button[@role='button'][2]")
            next_button.click()
        time.sleep(5)
        try:
            logger.info("输入手机号验证...")
            driver.find_element_by_xpath("//input[@name='text']").send_keys("+8619962025641")
            driver.find_element_by_xpath("//button[@data-testid='ocfEnterTextNextButton']").click()
            time.sleep(5)
        except Exception:
            logger.info("无需输入手机号验证")
        driver.find_element_by_xpath("//input[@name='password']").send_keys(login_user['pwd'])
        driver.find_element_by_xpath("//button[@data-testid='LoginForm_Login_Button']").click()
        time.sleep(5)
        try:
            driver.find_element_by_xpath("//button[@data-testid='confirmationSheetConfirm']").click()
            time.sleep(10)
        except:
            time.sleep(5)
        cookies = driver.get_cookies()
        # cookies = json.loads(response.text)['cookies']
        # 取cookie中的ct0为x-csrf-token，取gt为x-guest-token
        self.cookie_dict = {}
        for cookie in cookies:
            self.cookie_dict[cookie['name']] = cookie['value']
        self.header = {
            'Host': 'api.twitter.com',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0',
            'Accept': '*/*',
            'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
            'content-type': 'application/json',
            'authorization': self.authorization,
            # 'x-twitter-active-user': 'yes',
            'Origin': 'https://twitter.com',
            'Connection': 'keep-alive',
            'X-Csrf-Token': self.cookie_dict['ct0']
        }
        self.filter_key = self.settings['TWITTER_FILTER_KEY']
        self.pid_key = self.settings['TWITTER_PID_KEY']
        url_key = self.redis_client.get("MediaSpiders:Twitter_URL_Key").decode()
        account_query_api = self.settings['SOCIAL_USER_QUERY_ALL_API']
        account_query_api = account_query_api.format(sortBy="id", shuffleResult="true")
        post_data = {
            'userType': self.settings['TWITTER_USER_TYPE'],
            'userFlag': 0
        }
        account_rsp = json.loads(
            http_post(account_query_api, json.dumps(post_data), headers={"Content-Type": "application/json"}).text)
        all_user_info = []
        if account_rsp['code'] == 200:
            all_user_info = account_rsp['content']
        for user_info in all_user_info:
            graphql_url = f'https://twitter.com/i/api/graphql/{url_key}/UserTweets?variables=%7B%22userId%22%3A%22{user_info["userUid"]}%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22rweb_lists_timeline_redesign_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Afalse%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_media_download_video_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticleRichContentState%22%3Afalse%7D'
            yield scrapy.Request(graphql_url, callback=self.parse,
                                 meta={'uid': user_info['userUid'], 'currentCount': 0},
                                 cookies=self.cookie_dict, headers=self.header)

    def parse(self, response):
        uid = response.request.meta['uid']
        current_count = response.request.meta['currentCount']
        if current_count > 0:
            self.logger.info("翻页采集：第%s页" % int(current_count / 20 + 1))
        else:
            self.logger.info("首页采集")
        try:
            rsp = json.loads(response.text)
            entries = []
            try:
                instructions = rsp['data']['user']['result']['timeline']['timeline']['instructions']
            except:
                instructions = rsp['data']['user']['result']['timeline_v2']['timeline']['instructions']
            for i in instructions:
                if 'entries' in i:
                    entries = i['entries']
                    break
            if len(entries) > 0:
                for entry in entries:
                    if entry['entryId'].startswith('tweet'):
                        if 0 < self.total_num <= current_count:
                            return
                        current_count += 1
                        item = MediaspidersItem()
                        result = entry['content']['itemContent']['tweet_results']['result']
                        item['es_userid'] = result['core']['user_results']['result']['rest_id']
                        item['es_hkey'] = result['rest_id']
                        item['es_district'] = result['core']['user_results']['result']['legacy']['location']
                        screen_name = result['core']['user_results']['result']['legacy']['screen_name']
                        user_name = result['core']['user_results']['result']['legacy']['name']
                        item['es_urlname'] = 'https://twitter.com/%s/status/%s' % (screen_name, result['rest_id'])
                        item['es_authors'] = screen_name
                        item['es_extname'] = user_name
                        legacy = result['legacy']
                        author_full_text = legacy['full_text']
                        created_at = legacy['created_at']
                        item['es_commentcount'] = legacy['reply_count']
                        item['es_forwardcount'] = legacy['retweet_count']
                        item['es_likecount'] = legacy['favorite_count']
                        item['es_lasttime'] = get_current_timestamp()
                        item['es_loadtime'] = get_current_timestamp()
                        item['es_urltime'] = get_time_stamp(
                            str(created_at)) + 8 * 3600 * 1000  # TW默认使用的是零时区，转换为北京时间
                        if 'quoted_status_result' in result:
                            item['es_isrepost'] = 'yes'
                            item['es_urltitle'] = author_full_text
                            item['es_catalog1'] = author_full_text
                            legacy = result['quoted_status_result']['result']['legacy']
                            self.logger.info('采集引用推文原文信息')
                        elif 'retweeted_status_result' in legacy:
                            item['es_isrepost'] = 'yes'
                            legacy = legacy['retweeted_status_result']['result']['legacy']
                            self.logger.info('采集转发推文原文信息')
                        item['es_content'] = legacy['full_text']
                        item['es_urlcontent'] = legacy['full_text']
                        # 下载图片
                        image_url_list = []
                        if 'entities' in legacy and 'media' in legacy['entities']:
                            medias = legacy['entities']['media']
                            for media in medias:
                                if media['type'] == 'photo':
                                    image_url_list.append(media['media_url_https'])
                        # 下载视频
                        video_url_list = []
                        if 'extended_entities' in legacy and 'media' in legacy['extended_entities']:
                            ex_medias = legacy['extended_entities']['media']
                            for media in ex_medias:
                                if media['type'] == 'video':
                                    variants = media['video_info']['variants']
                                    target_url = ''
                                    bitrate_min = 0
                                    # 筛选清晰度最低的视频
                                    for v in variants:
                                        if v['content_type'] == 'video/mp4':
                                            if bitrate_min > 0 and v['bitrate'] > bitrate_min:
                                                continue
                                            else:
                                                target_url = v['url']
                                                bitrate_min = v['bitrate']
                                    if '?' in target_url:
                                        target_url = target_url.split('?')[0]
                                    video_url_list.append(target_url)
                                    self.logger.info('获取到视频，URL为%s' % target_url)
                        item['image_urls'] = image_url_list
                        item['file_urls'] = video_url_list
                        item['es_sitename'] = 'twitter'
                        item['es_srcname'] = 'twitter'
                        item['es_carriertype'] = 'media'
                        # 对于48小时前发布的内容，加入判重
                        if item['es_lasttime'] - item['es_urltime'] > 48 * 3600 * 1000:
                            response.request.meta['CUSTOM_FILTER_KEY'] = self.filter_key
                            response.request.meta['CUSTOM_FILTER_VAL'] = legacy['id_str']
                        yield item
        except:
            self.logger.error("解析response错误")
        try:
            for entry in entries:
                if entry['entryId'].find('cursor-bottom') >= 0:
                    cursor = entry['content']['value']
                    url = response.url
                    index = url.find('variables=') + 10
                    variables = url[index:]
                    body = parse.unquote(variables)
                    pattern = re.compile('("cursor":".+IAAA",)*"includePromotedContent":true,')
                    out = re.sub(pattern, f'"cursor":"{cursor}","includePromotedContent":true,', body)
                    next_url = url[:index] + parse.quote(out)
                    yield scrapy.Request(next_url, callback=self.parse,
                                         meta={'uid': uid, 'currentCount': current_count},
                                         headers=self.header, cookies=self.cookie_dict)
                    break
        except Exception as error:
            self.logger.error('获取cursor失败', error)
