import json
import logging as logger
import time
import uuid
import redis

import scrapy
import validators
from selenium.common.exceptions import ElementClickInterceptedException
from selenium.webdriver.common.keys import Keys

from ForumSpiders.items import EsItem
from ForumSpiders.scrapy_selenium import SeleniumRequest
from ForumSpiders.utils.date_util import get_time_stamp
from ForumSpiders.utils.time_util import pause_for_random


# 根据问题ID获取其下所有的回答
class ZhihuPageSpider(scrapy.Spider):
    name = 'zhihu_page'
    custom_settings = {
        'ITEM_PIPELINES': {
            'scrapy.pipelines.images.ImagesPipeline': 2,
            'ForumSpiders.pipelines.ProtobufSavePipeline': 300
        },
        'SPIDER_MIDDLEWARES': {
            'ForumSpiders.middlewares.DumpFilterSpiderMiddleware': 600
        }
    }

    def __init__(self, params=None, *args, **kwargs):
        super(ZhihuPageSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        self.qids = []
        self.max_answer_count = 10
        if 'qids' in json_params:
            self.qids = json_params['qids']
        self.redis_client = None

    def start_requests(self):
        self.max_answer_count = self.settings['ZHIHU_MAX_ANSWER_COUNT']
        self.redis_client = redis.Redis(host=self.settings['REDIS_HOST'], port=self.settings['REDIS_PORT'],
                                        password=self.settings['REDIS_PWD'])
        yield SeleniumRequest(url='https://www.zhihu.com', callback=self.parse)

    def parse(self, response, **kwargs):
        driver = response.request.meta['driver']
        driver.maximize_window()
        search_keys = self.redis_client.smembers('ForumSpiders:zhihu_search_keywords')
        for search_key in search_keys:
            search_url = 'https://www.zhihu.com/search?type=content&q={}'.format(search_key.decode())
            driver.get(search_url)
            time.sleep(8)
            question_uris = driver.find_elements_by_xpath("//div[@class='ContentItem AnswerItem']/h2")
            for uri in question_uris:
                title = uri.text
                href = uri.find_element_by_xpath('.//a').get_attribute('href')
                qid_i_start = href.find('question/') + len('question/')
                qid_i_end = href.find('/answer')
                qid = href[qid_i_start:qid_i_end]
                logger.info("获取到问题 qid: {} - {}".format(qid, title))
                self.qids.append(qid)

        for qid in self.qids:
            url = 'https://www.zhihu.com/question/{}'.format(qid)
            logger.info("start to get qid: {}".format(qid))
            driver.get(url)
            pause_for_random(3)
            close_button = driver.find_element_by_xpath("//button[@class='Button Modal-closeButton Button--plain']")
            if close_button is not None:
                close_button.click()
            # 解析问题描述
            question_area = driver.find_element_by_xpath(
                "//div[@data-za-detail-view-path-module='QuestionDescription']")
            question_date_created = driver.find_element_by_xpath("//meta[@itemprop='dateCreated']").get_attribute(
                "content")
            question_date_modified = driver.find_element_by_xpath("//meta[@itemprop='dateModified']").get_attribute(
                "content")
            question_id = json.loads(question_area.get_attribute("data-za-extra-module"))['card']['content']['token']
            question_title = question_area.find_element_by_xpath(".//h1").text
            try:
                question_expand_button = question_area.find_element_by_xpath(
                    ".//button[contains(@class, 'QuestionRichText-more')]")
                question_expand_button.click()
                question_description = question_area.find_element_by_xpath(
                    ".//div[@class='QuestionRichText QuestionRichText--expandable']").text
            except ElementClickInterceptedException:
                logger.error("展开问题描述的按钮被遮挡，改为获取缩略的描述")
                question_description = question_area.find_element_by_xpath(
                    ".//div[@class='QuestionRichText QuestionRichText--expandable QuestionRichText--collapsed']").text
            except Exception as e:
                logger.debug(repr(e))
                question_description = ''

            question_info = {
                "item_id": str(uuid.uuid4()).replace("-", ""),
                'question_id': question_id,
                'question_title': question_title,
                'question_link': "https://www.zhihu.com/question/{}".format(question_id),
                'date_created': question_date_created.replace("T", " ").replace("Z", ""),
                'date_modified': question_date_modified.replace("T", " ").replace("Z", ""),
                'content_text': question_description.replace("'", "\\'")
            }
            question_item = EsItem()
            question_item['es_sid'] = question_info['item_id']
            question_item['es_hkey'] = question_info['question_id']
            question_item['es_isrepost'] = '0'
            question_item['es_urlname'] = question_info['question_link']
            question_item['es_lasttime'] = get_time_stamp(question_info['date_modified'])
            question_item['es_urltime'] = get_time_stamp(question_info['date_created'])
            question_item['es_urltitle'] = question_info['question_title']
            question_item['es_urlcontent'] = question_info['content_text']
            question_item['es_doclength'] = len(question_info['content_text'])
            question_item['es_sitename'] = '知乎'
            question_item['es_srcname'] = "https://www.zhihu.com/"
            question_item['es_carriertype'] = 'forum'
            yield question_item
            # 翻页获取回答
            index = 0
            answer_list_item = driver.find_elements_by_xpath(
                '//div[@id="QuestionAnswers-answers"]//div[@class="List-item"]')
            while index < self.max_answer_count:
                pause_for_random(2)
                if index >= len(answer_list_item):
                    logger.info("已遍历 {} 条回答，进行翻页加载操作...".format(index))
                    driver.find_element_by_css_selector("body").send_keys(Keys.END)
                    pause_for_random(0.5)
                    driver.find_element_by_css_selector("body").send_keys(Keys.ARROW_UP)
                    driver.find_element_by_css_selector("body").send_keys(Keys.ARROW_DOWN)
                    driver.find_element_by_css_selector("body").send_keys(Keys.ARROW_DOWN)
                    driver.find_element_by_css_selector("body").send_keys(Keys.ARROW_DOWN)
                    pause_for_random(3)
                    answer_list_item = driver.find_elements_by_xpath(
                        '//div[@id="QuestionAnswers-answers"]//div[@class="List-item"]')
                    if index >= len(answer_list_item):
                        logger.info("已加载该问题下的全部 {} 条回答".format(len(answer_list_item)))
                        break
                try:
                    current_answer = answer_list_item[index]
                    current_answer_content = current_answer.find_element_by_xpath(
                        './/div[@class="ContentItem AnswerItem"]')
                    data_za_extra_module = current_answer_content.get_attribute("data-za-extra-module")
                    data_zop = current_answer_content.get_attribute("data-zop")
                    if data_za_extra_module is None:
                        logger.info("似乎遍历到了一条非回答的Item，跳过这一条")
                        index += 1
                        continue
                    data = json.loads(data_za_extra_module)
                    aid = json.loads(data_zop)['itemId']
                    answer_info = {
                        "item_id": str(uuid.uuid4()).replace("-", ""),
                        "content_type": "a",
                        "question_id": qid,
                        "answer_id": aid,
                        "question_title": question_title,
                        "question_link": "https://www.zhihu.com/question/{}".format(qid),
                        "answer_link": "https://www.zhihu.com/question/{}/answer/{}".format(qid, aid),
                        "author_name": json.loads(current_answer_content.get_attribute("data-zop"))['authorName'],
                        "author_link": current_answer_content.find_element_by_xpath(
                            '//meta[@itemprop="url"]').get_attribute('content'),
                        "upvote_num": data["card"]["content"]["upvote_num"],
                        "comment_num": data["card"]["content"]["comment_num"],
                        "date_created": "{}".format(
                            current_answer_content.find_element_by_class_name(
                                'ContentItem-time').find_element_by_tag_name(
                                'span').get_attribute('data-tooltip')).replace("发布于 ", "", -1)
                    }
                    content_text = current_answer_content.find_element_by_class_name('RichContent-inner').text
                    content_text_paragraph = content_text.split('\n')
                    answer_info['content_text'] = '<br/>\r\n'.join(content_text_paragraph)
                    imgs = current_answer_content.find_element_by_class_name(
                        'RichContent-inner').find_elements_by_tag_name('img')
                    answer_info['image_urls'] = []
                    for img in imgs:
                        img_url = img.get_attribute('src')
                        if validators.url(img_url):
                            answer_info['image_urls'].append(img_url)
                    answer_info['author_id'] = answer_info['author_link'].split('/')[-1]
                    answer_item = EsItem()
                    answer_item['es_sid'] = answer_info['item_id']
                    answer_item['es_hkey'] = answer_info['answer_id']
                    answer_item['es_urlname'] = answer_info['answer_link']
                    answer_item['es_urltitle'] = answer_info['question_title']
                    answer_item['es_isrepost'] = '1'
                    answer_item['es_urltopic'] = answer_info['question_id']
                    answer_item['es_lasttime'] = get_time_stamp(answer_info['date_created'])
                    answer_item['es_urltime'] = get_time_stamp(answer_info['date_created'])
                    answer_item['es_authors'] = answer_info['author_name']
                    answer_item['es_urlimage'] = answer_info['image_urls']
                    answer_item['es_urlcontent'] = answer_info['content_text']
                    answer_item['es_carriertype'] = 'forum'
                    answer_item['es_sitename'] = "知乎"
                    answer_item['es_srcname'] = "https://www.zhihu.com/"
                    answer_item['es_likecount'] = answer_info['upvote_num']
                    yield answer_item
                except Exception as e:
                    logger.error(e)
                index += 1
            if index >= self.max_answer_count:
                logger.info("最多获取 {} 个回答，已经全部获取到。".format(self.max_answer_count))
