# -*- coding: utf-8 -*-
from datetime import datetime
from datetime import timezone, timedelta
import json
import logging as logger
import random
import re
import time
from urllib import parse

import redis
import scrapy
from scrapy_selenium import SeleniumRequest
from MediaSpiders.items import MediaspidersItem, TwitterUserInfoItem
from MediaSpiders.utils.http_utils import http_post
from MediaSpiders.utils.login_utils import login
from MediaSpiders.utils.time_utils import get_time_stamp, get_current_timestamp


def form_cookie_dict(cookie_string):
    cookie_string_list = cookie_string.split(';')
    cookie_dict = {}
    for cookie in cookie_string_list:
        key = cookie.split('=')[0].replace(' ', '')
        cookie_dict[key] = cookie.split('=')[1]
    return cookie_dict


class TwitterSpider(scrapy.Spider):
    name = 'TwitterUserInfoSpider'
    custom_settings = {

        'PROTO_SAVE_FILE_NAME': 'public_twitter_user_info_data_',
        'IMAGES_STORE': r'/usr/local/temp_image/twitteruserinfo',
        'IMAGES_RESULT_FIELD': 'avatar_path',
        'ITEM_PIPELINES': {
            'scrapy.pipelines.images.ImagesPipeline': 2,
            'MediaSpiders.pipelines.TwitterUserDataSaveToMySQL': 300,
        },
        'SPIDER_MIDDLEWARES': {
            'MediaSpiders.middlewares.DumpFilterSpiderMiddleware': 543,
            'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': None
        }

    }

    def __init__(self, params=None, *args, **kwargs):
        super(TwitterSpider, self).__init__(*args, **kwargs)
        self.total_num = 100
        self.authorization = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'
        if params:
            json_params = json.loads(params)
            if 'totalNum' in json_params:
                self.total_num = int(json_params['totalNum'])
            if 'authorization' in json_params:
                self.authorization = json_params['authorization']
            if 'job_id' in json_params:
                self.job_id = json_params['job_id']

    def start_requests(self):
        yield SeleniumRequest(url='https://www.google.com/', callback=self.login_twitter)

    def login_twitter(self, response):
        self.redis_client = redis.Redis(host=self.settings['REDIS_HOST'], port=self.settings['REDIS_PORT'],
                                        password=self.settings['REDIS_PWD'])
        self.simhash_filter_key = self.settings['TWITTER_SIMHASH_FILTER_KEY']
        cookie_string = None
        # 获取采集登录账号并登录
        login_users = self.redis_client.smembers('MediaSpiders:Twitter_login_accounts')
        # 尝试自动化登录网页获取 cookies，若失败则从redis中 使用已有cookies
        # try:
        #
        #     driver = login().login_with_selenium(
        #         'https://x.com/i/flow/login',
        #         self.name,
        #         login_users=login_users,
        #         response=response
        #     )
        #     cookies = driver.get_cookies()
        #     # 取cookie中的ct0为x-csrf-token，取gt为x-guest-token
        #     self.cookie_dict = {}
        #     for cookie in cookies:
        #         self.cookie_dict[cookie['name']] = cookie['value']
        # except Exception as e:
        #     logger.info("自动化获取cookies失败")
        cookie_string = self.redis_client.get("MediaSpiders:Twitter_Cookies").decode()
        self.cookie_dict = form_cookie_dict(cookie_string)

        ct0 = self.cookie_dict.get('ct0')
        if not ct0:
            logger.error("redis中cookie缺失ct0 (CSRF token)！")
            return
        self.header = {
            'Host': 'api.twitter.com',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36 Edg/143.0.0.0',
            'Accept': '*/*',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
            'content-type': 'application/json',
            'authorization': self.authorization,
            'Origin': 'https://twitter.com',
            'Cookie': cookie_string,
            'X-Csrf-Token': ct0
        }
        self.filter_key = self.settings['TWITTER_FILTER_KEY']
        self.pid_key = self.settings['TWITTER_PID_KEY']
        url_key = self.redis_client.get("MediaSpiders:Twitter_URL_Key").decode()
        account_query_api = self.settings['SOCIAL_USER_QUERY_ALL_API']
        account_query_api = account_query_api.format(sortBy="id", shuffleResult="true")
        post_data = {
            'userType': self.settings['TWITTER_USER_TYPE'],
            'userFlag': 0
        }
        account_rsp = json.loads(
            http_post(account_query_api, json.dumps(post_data), headers={"Content-Type": "application/json"}).text)
        all_user_info = []
        if account_rsp['code'] == 200:
            all_user_info = account_rsp['content']
        for user_info in all_user_info:
            graphql_url = f'https://x.com/i/api/graphql/-oaLodhGbbnzJBACb1kk2Q/UserByScreenName?variables=%7B%22screen_name%22%3A%22{user_info["userName"]}%22%2C%22withGrokTranslatedBio%22%3Afalse%7D&features=%7B%22hidden_profile_subscriptions_enabled%22%3Atrue%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22subscriptions_verification_info_is_identity_verified_enabled%22%3Atrue%2C%22subscriptions_verification_info_verified_since_enabled%22%3Atrue%2C%22highlights_tweets_tab_ui_enabled%22%3Atrue%2C%22responsive_web_twitter_article_notes_tab_enabled%22%3Atrue%2C%22subscriptions_feature_can_gift_premium%22%3Atrue%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D&fieldToggles=%7B%22withPayments%22%3Afalse%2C%22withAuxiliaryUserLabels%22%3Atrue%7D'
            yield scrapy.Request(url=graphql_url, callback=self.parse,
                                 meta={
                                     'uid': user_info['userUid'],
                                     'uname': user_info['userName'],
                                     'proxy': 'http://127.0.0.1:10809',
                                     'currentCount': 0
                                 },
                                 cookies=self.cookie_dict, headers=self.header)

    def parse(self, response):
        uid = response.request.meta['uid']
        uname = response.request.meta['uname']
        current_count = response.request.meta['currentCount']
        if current_count > 0:
            self.logger.info("翻页采集：第%s页" % int(current_count / 20 + 1))
        else:
            self.logger.info("首页采集")
        try:
            rsp = json.loads(response.text)
            entries = []
            instructions = rsp['data']['user']['result']
            item = TwitterUserInfoItem()
            item['crawl_time'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
            item['is_newest'] = 1
            item['platform_type'] = "Twitter"
            item['user_id'] = instructions['rest_id']
            item['nickname'] = instructions['core']['name']
            item['username'] = instructions['core']['screen_name']
            item['user_url'] = f'https://x.com/{uname}'
            item['avatar_url'] = instructions['avatar']['image_url']
            item['image_urls'] = [instructions['avatar']['image_url']]
            item['intro'] = instructions['legacy']['description']
            item['city'] = instructions.get('legacy', {}).get('location', {}).get('location', '').strip()
            try:
                # 转换为 datetime 对象
                ts = get_time_stamp(
                    str(instructions['core']['created_at'])) + 8 * 3600 * 1000
                dt = datetime.fromtimestamp(ts / 1000, tz=timezone.utc)
                item['join_date'] = dt.strftime('%Y-%m-%d %H:%M:%S')  # '2012-06-28 12:25:01'
            except (ValueError, KeyError) as e:
                item['join_date'] = None  # 或记录日志
                logger.error('时间转换失败：' + e)
            item['post_count'] = instructions['legacy']['statuses_count']
            item['follow_count'] = instructions['legacy']['friends_count']
            item['fans_count'] = instructions['legacy']['followers_count']
            item['is_verified'] = str(instructions['is_blue_verified'])
            verified_type = instructions.get('verification', {}).get('verified_type', None)  # 认证类型
            yield item
        except:
            self.logger.error("解析response错误")
