# -*- coding: utf-8 -*-
import json
import logging as logger
import time

import scrapy
from redisbloom.client import Client
from scrapy.utils.project import get_project_settings
from scrapy_selenium import SeleniumRequest

from MediaSpiders.items import MediaspidersItem
from MediaSpiders.utils.date_utils import get_time_stamp
from MediaSpiders.utils.http_utils import http_post


class YoutubeSpider(scrapy.Spider):
    name = 'YoutubeSpider'
    custom_settings = {
        'PROTO_MODULE_PATH': 'MediaSpiders.proto.Es_pb2',
        'PROTO_CLASS_NAME': 'EsSets',
        'PROTO_FIELD_NAME': 'Es',
        'PROTO_SAVE_FILE_NAME': 'public_info_data_',
        'IMAGES_STORE': r'/usr/local/temp_image/youtube',
        'IMAGES_RESULT_FIELD': 'es_urlimage',
        'ZIP_FILE_NAME': 'image_data_publicinfo_',
        'FILE_ZIP_FILE_NAME': 'image_data_plane_',
        'ITEM_PIPELINES': {
            'scrapy.pipelines.images.ImagesPipeline': 2,
            'MediaSpiders.pipelines.ProtobufSavePipeline': 300
        },
        'DOWNLOAD_DELAY': 2,
        'SPIDER_MIDDLEWARES': {
            'MediaSpiders.middlewares.DumpFilterSpiderMiddleware': 543,
        }
    }
    settings = get_project_settings()

    def __init__(self, params=None, *args, **kwargs):
        super(YoutubeSpider, self).__init__(*args, **kwargs)
        self.bloom_filter = None
        if params:
            json_params = json.loads(params)
            logger.info(json_params)
            if 'job_id' in json_params:
                self.job_id = json_params['job_id']

    def start_requests(self):
        self.bloom_filter = Client(host=self.settings['REDIS_HOST'], port=self.settings['REDIS_PORT'],
                                   password=self.settings['REDIS_PWD'])
        self.simhash_filter_key = self.settings['YOUTUBE_SIMHASH_FILTER_KEY']
        yield SeleniumRequest(url='https://www.google.com', callback=self.parse)

    def parse(self, response):
        driver = response.request.meta['driver']
        driver.maximize_window()
        account_query_api = self.settings['SOCIAL_USER_QUERY_ALL_API']
        account_query_api = account_query_api.format(sortBy="id", shuffleResult="true")
        post_data = {
            'userType': self.settings['YOUTUBE_USER_TYPE'],
            'userFlag': 0
        }
        account_rsp = json.loads(
            http_post(account_query_api, json.dumps(post_data), headers={"Content-Type": "application/json"}).text)
        all_youtube_user_info = []
        if account_rsp['code'] == 200:
            all_youtube_user_info = account_rsp['content']
        for user_line in all_youtube_user_info:
            user_name = user_line['userName']
            user_id = user_line['userUid']
            current_url = f'https://www.youtube.com/@{user_id}/videos'
            driver.get(current_url)
            # driver.execute_script('window.scrollTo(0, document.body.scrollHeight)')
            time.sleep(5)
            logger.info("Current URL: {}".format(current_url))
            try:
                current_page_articles_links = driver.find_elements_by_xpath(
                    "//div[@id='content']/ytd-rich-grid-media//ytd-thumbnail/a[@id='thumbnail']")
                detail_links = []
                for link_item in current_page_articles_links:
                    href = link_item.get_attribute("href")
                    detail_links.append(href)
                ytb_dl_api_data_body = {
                    'urls': detail_links
                }
                post_data_body = json.dumps(ytb_dl_api_data_body, ensure_ascii=False)
                # dl_rsp = http_post('http://107.182.191.3:28081/download', post_data_body)
                # logger.info(dl_rsp.content.decode())
                for link in detail_links:
                    if self.bloom_filter.bfAdd(self.settings['YOUTUBE_FILTER_KEY'], link) <= 0:
                        logger.info("跳过已采集内容")
                        continue
                    driver.get(link)
                    time.sleep(2)
                    detail_item = self.get_details(driver, user_id, user_name)
                    yield detail_item
            except Exception as e:
                logger.error(repr(e))

    def get_details(self, driver, user_uid, user_name):
        try:
            page_url = driver.find_element_by_xpath("//meta[@property='og:url']").get_attribute("content")
            title = driver.find_element_by_xpath("//meta[@property='og:title']").get_attribute("content")
            upload_date = driver.find_element_by_xpath("//meta[@itemprop='uploadDate']").get_attribute("content")
            expand_button = driver.find_elements_by_xpath("//tp-yt-paper-button[@id='expand']")
            if len(expand_button) > 0:
                expand_button[0].click()
            description = ''
            description_areas = driver.find_elements_by_xpath(
                "//div[@id='description']//yt-attributed-string//span[@class='yt-core-attributed-string--link-inherit-color']")
            for description_area in description_areas:
                description += description_area.text
            page_item = MediaspidersItem()
            page_item['es_urltitle'] = title
            page_item['es_urlcontent'] = description
            page_item['es_urlname'] = page_url
            page_item['es_video'] = f'video/youtube/{page_url[32:]}.mp4'
            page_item['es_urltime'] = get_time_stamp(upload_date)
            page_item['es_lasttime'] = page_item['es_urltime']
            page_item['es_loadtime'] = str(int(time.time() * 1000))
            page_item['es_authors'] = user_uid
            page_item['es_extname'] = user_name  # 显示的作者名
            page_item['es_sitename'] = 'youtube'
            page_item['es_srcname'] = 'https://www.youtube.com'
            page_item['es_carriertype'] = 'video'
            return page_item
        except Exception as e:
            logger.error(repr(e))
            return
