# -*- coding: utf-8 -*-
import json
import logging as logger
import random
import re
import time
from urllib import parse

import redis
import scrapy
from scrapy_selenium import SeleniumRequest
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from MediaSpiders.items import MediaspidersItem
from MediaSpiders.utils.http_utils import http_post
from MediaSpiders.utils.login_utils import login
from MediaSpiders.utils.time_utils import get_time_stamp, get_current_timestamp
from selenium.webdriver.common.action_chains import ActionChains

from MediaSpiders.utils.traslate_utils import translate_single, translate_content_with_paragraphs, needs_translation


def form_cookie_dict(cookie_string):
    cookie_string_list = cookie_string.split(';')
    cookie_dict = {}
    for cookie in cookie_string_list:
        key = cookie.split('=')[0].replace(' ', '')
        cookie_dict[key] = cookie.split('=')[1]
    return cookie_dict


class TwitterSpider(scrapy.Spider):
    name = 'TwitterUserSpider'
    custom_settings = {
        'PROTO_MODULE_PATH': 'MediaSpiders.proto.Es_pb2',
        'PROTO_CLASS_NAME': 'EsSets',
        'PROTO_FIELD_NAME': 'Es',
        'PROTO_SAVE_FILE_NAME': 'public_info_data_',
        'IMAGES_STORE': r'/usr/local/temp_image/twitter',
        'IMAGES_RESULT_FIELD': 'es_urlimage',
        'FILES_STORE': r'/usr/local/videos',
        'FILES_RESULT_FIELD': 'es_video',
        'ZIP_FILE_NAME': 'image_data_publicinfo_',
        'FILE_ZIP_FILE_NAME': 'image_data_plane_',
        'ITEM_PIPELINES': {
            'scrapy.pipelines.images.ImagesPipeline': 2,
            'scrapy.pipelines.files.FilesPipeline': 1,
            'MediaSpiders.pipelines.ProtobufSavePipeline': 300,
        },
        'SPIDER_MIDDLEWARES': {
            'MediaSpiders.middlewares.DumpFilterSpiderMiddleware': 543,
            # 'MediaSpiders.middlewares.KeywordFilterSpiderMiddleware': 544,
            # 'MediaSpiders.middlewares.SimhashFilterSpiderMiddleware': 545,
            # 'scrapy_splash.SplashDeduplicateArgsMiddleware': 700,
            'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': None
        }

    }

    def __init__(self, params=None, *args, **kwargs):
        super(TwitterSpider, self).__init__(*args, **kwargs)
        self.total_num = 100
        self.authorization = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'
        if params:
            json_params = json.loads(params)
            if 'totalNum' in json_params:
                self.total_num = int(json_params['totalNum'])
            if 'authorization' in json_params:
                self.authorization = json_params['authorization']
            if 'job_id' in json_params:
                self.job_id = json_params['job_id']

    def start_requests(self):
        yield SeleniumRequest(url='https://www.google.com/', callback=self.login_twitter)

    def login_twitter(self, response):
        self.redis_client = redis.Redis(host=self.settings['REDIS_HOST'], port=self.settings['REDIS_PORT'],
                                        password=self.settings['REDIS_PWD'])
        self.simhash_filter_key = self.settings['TWITTER_SIMHASH_FILTER_KEY']
        cookie_string = None
        # 获取采集登录账号并登录
        login_users = self.redis_client.smembers('MediaSpiders:Twitter_login_accounts')
        # 尝试自动化登录网页获取 cookies，若失败则从redis中 使用已有cookies
        try:

            driver = login().login_with_selenium(
                'https://x.com/i/flow/login',
                self.name,
                login_users=login_users,
                response=response
            )
            cookies = driver.get_cookies()
            # 取cookie中的ct0为x-csrf-token，取gt为x-guest-token
            self.cookie_dict = {}
            for cookie in cookies:
                self.cookie_dict[cookie['name']] = cookie['value']
        except Exception as e:
            logger.info("自动化获取cookies失败")
            cookie_string = self.redis_client.get("MediaSpiders:Twitter_Cookies").decode()
            self.cookie_dict = form_cookie_dict(cookie_string)
            # 5. 构建 headers
            ct0 = self.cookie_dict.get('ct0')
            if not ct0:
                logger.error("redis中cookie缺失ct0 (CSRF token)！")
                return

        self.header = {
            'Host': 'api.twitter.com',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36 Edg/143.0.0.0',
            'Accept': '*/*',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
            'content-type': 'application/json',
            'authorization': self.authorization,
            'Origin': 'https://twitter.com',
            'Cookie': cookie_string,
            'X-Csrf-Token': ct0
        }
        self.filter_key = self.settings['TWITTER_FILTER_KEY']
        self.pid_key = self.settings['TWITTER_PID_KEY']
        url_key = self.redis_client.get("MediaSpiders:Twitter_URL_Key").decode()
        account_query_api = self.settings['SOCIAL_USER_QUERY_ALL_API']
        account_query_api = account_query_api.format(sortBy="id", shuffleResult="true")
        post_data = {
            'userType': self.settings['TWITTER_USER_TYPE'],
            'userFlag': 0
        }
        account_rsp = json.loads(
            http_post(account_query_api, json.dumps(post_data), headers={"Content-Type": "application/json"}).text)
        all_user_info = []
        if account_rsp['code'] == 200:
            all_user_info = account_rsp['content']
        for user_info in all_user_info:
            graphql_url = f'https://x.com/i/api/graphql/{url_key}/UserTweets?variables=%7B%22userId%22%3A%22{user_info["userUid"]}%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Atrue%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22responsive_web_grok_annotations_enabled%22%3Afalse%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Afalse%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22post_ctas_fetch_enabled%22%3Afalse%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_imagine_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_community_note_auto_translation_is_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D'

            yield scrapy.Request(url=graphql_url, callback=self.parse,
                                 meta={
                                     'uid': user_info['userUid'],
                                     'proxy': 'http://127.0.0.1:10809',
                                     'currentCount': 0
                                 },
                                 cookies=self.cookie_dict, headers=self.header)

    def parse(self, response):
        uid = response.request.meta['uid']
        current_count = response.request.meta['currentCount']
        if current_count > 0:
            self.logger.info("翻页采集：第%s页" % int(current_count / 20 + 1))
        else:
            self.logger.info("首页采集")
        try:
            rsp = json.loads(response.text)
            entries = []
            try:
                instructions = rsp['data']['user']['result']['timeline']['timeline']['instructions']
            except:
                instructions = rsp['data']['user']['result']['timeline_v2']['timeline']['instructions']
            for i in instructions:
                if 'entries' in i:
                    entries = i['entries']
                    break
            if len(entries) > 0:
                for entry in entries:
                    if entry['entryId'].startswith('tweet'):
                        if 0 < self.total_num <= current_count:
                            return
                        current_count += 1
                        item = MediaspidersItem()
                        result = entry['content']['itemContent']['tweet_results']['result']
                        item['es_userid'] = result['core']['user_results']['result']['rest_id']
                        item['es_hkey'] = result['rest_id']
                        item['es_district'] = result['core']['user_results']['result']['location']
                        screen_name = result['core']['user_results']['result']['core']['screen_name']
                        user_name = result['core']['user_results']['result']['core']['name']
                        item['es_urlname'] = 'https://x.com/%s/status/%s' % (screen_name, result['rest_id'])
                        item['es_authors'] = screen_name
                        item['es_extname'] = user_name
                        legacy = result['legacy']
                        author_full_text = legacy['full_text']
                        created_at = legacy['created_at']
                        item['es_commentcount'] = legacy['reply_count']
                        item['es_forwardcount'] = legacy['retweet_count']
                        item['es_likecount'] = legacy['favorite_count']
                        item['es_lasttime'] = get_current_timestamp()
                        item['es_loadtime'] = get_current_timestamp()
                        item['es_urltime'] = get_time_stamp(
                            str(created_at)) + 8 * 3600 * 1000  # TW默认使用的是零时区，转换为北京时间
                        if 'quoted_status_result' in result:
                            item['es_isrepost'] = 1
                            item['es_urltitle'] = author_full_text
                            item['es_catalog1'] = author_full_text
                            # 判断是否需要翻译
                            if needs_translation(author_full_text):
                                item['es_content'] = translate_single(author_full_text)  # TODO 翻译
                            else:
                                item['es_content'] = url_content
                            legacy = result['quoted_status_result']['result']['legacy']
                            self.logger.info('采集引用推文原文信息')
                        elif 'retweeted_status_result' in legacy:
                            item['es_isrepost'] = 1
                            legacy = legacy['retweeted_status_result']['result']['legacy']
                            self.logger.info('采集转发推文原文信息')
                        item['es_urlcontent'] = legacy['full_text']
                        # 获取文本
                        url_content = legacy['full_text']
                        # 判断是否需要翻译
                        if needs_translation(url_content):
                            item['es_content'] = translate_content_with_paragraphs(url_content)  # TODO 翻译
                        else:
                            item['es_content'] = url_content
                        # 下载图片
                        image_url_list = []
                        if 'entities' in legacy and 'media' in legacy['entities']:
                            medias = legacy['entities']['media']
                            for media in medias:
                                if media['type'] == 'photo':
                                    image_url_list.append(media['media_url_https'])
                        # 下载视频
                        video_url_list = []
                        if 'extended_entities' in legacy and 'media' in legacy['extended_entities']:
                            ex_medias = legacy['extended_entities']['media']
                            for media in ex_medias:
                                if media['type'] == 'video':
                                    variants = media['video_info']['variants']
                                    target_url = ''
                                    bitrate_min = 0
                                    # 筛选清晰度最低的视频
                                    for v in variants:
                                        if v['content_type'] == 'video/mp4':
                                            if bitrate_min > 0 and v['bitrate'] > bitrate_min:
                                                continue
                                            else:
                                                target_url = v['url']
                                                bitrate_min = v['bitrate']
                                    if '?' in target_url:
                                        target_url = target_url.split('?')[0]
                                    video_url_list.append(target_url)
                                    self.logger.info('获取到视频，URL为%s' % target_url)
                        item['image_urls'] = image_url_list
                        item['file_urls'] = video_url_list
                        item['es_sitename'] = 'twitter'
                        item['es_srcname'] = 'twitter'
                        item['es_carriertype'] = 'media'
                        # 对于48小时前发布的内容，加入判重
                        if item['es_lasttime'] - item['es_urltime'] > 48 * 3600 * 1000:
                            response.request.meta['CUSTOM_FILTER_KEY'] = self.filter_key
                            response.request.meta['CUSTOM_FILTER_VAL'] = legacy['id_str']
                        yield item
        except:
            self.logger.error("解析response错误")
        try:
            for entry in entries:
                if entry['entryId'].find('cursor-bottom') >= 0:
                    cursor = entry['content']['value']
                    url = response.url
                    index = url.find('variables=') + 10
                    variables = url[index:]
                    body = parse.unquote(variables)
                    pattern = re.compile('("cursor":".+IAAA",)*"includePromotedContent":true,')
                    out = re.sub(pattern, f'"cursor":"{cursor}","includePromotedContent":true,', body)
                    next_url = url[:index] + parse.quote(out)
                    yield scrapy.Request(next_url, callback=self.parse,
                                         meta={'uid': uid, 'currentCount': current_count},
                                         headers=self.header, cookies=self.cookie_dict)
                    break
        except Exception as error:
            self.logger.error('获取cursor失败', error)
