import json
import logging as logger
import time

import scrapy

from ForumSpiders.items import EsItem
from ForumSpiders.utils.date_util import get_time_stamp
from ForumSpiders.utils.string_util import get_str_md5
from ForumSpiders.utils.website_util import buildUrl


class MdcTwSpider(scrapy.Spider):
    name = 'mdc_tw'
    custom_settings = {
        "DOWNLOADER_MIDDLEWARES": {
            'ForumSpiders.middlewares.UserAgentDownloaderMiddleware': 545,
            'ForumSpiders.middlewares.RandomDelayDownloaderMiddleware': 999,
        },
        'ITEM_PIPELINES': {
            'scrapy.pipelines.images.ImagesPipeline': 2,
            'ForumSpiders.pipelines.ProtobufSavePipeline': 300
        },
        'SPIDER_MIDDLEWARES': {
            'ForumSpiders.middlewares.DumpFilterSpiderMiddleware': 600
        },
        'ITEM_BATCH_SAVE_NUMBER': 20
    }
    target_home_page_url = ['http://www.acewings.com/cobrachen/forum/forum.asp?FORUM_ID=14']

    def __init__(self, params=None, *args, **kwargs):
        super(MdcTwSpider, self).__init__(*args, **kwargs)
        self.proxy = None
        if params is not None:
            json_params = json.loads(params)
            logger.info(json_params)
            if "proxy" in json_params:
                self.proxy = json_params["proxy"]
            if 'job_id' in json_params:
                self.job_id = json_params['job_id']

    def start_requests(self):
        # urls = ['http://www.acewings.com/cobrachen/forum/topic.asp?TOPIC_ID=11824']
        request_meta = {
            "info": "start_page",
            # "current_page_num": 1
        }
        if self.proxy is not None:
            request_meta["proxy"] = self.proxy
        # for url in urls:
        #     yield scrapy.Request(url=url,
        #                          callback=self.parse,
        #                          meta=request_meta)
        for url in self.target_home_page_url:
            yield scrapy.Request(url=url,
                                 callback=self.next_requests,
                                 meta=request_meta)

    def next_requests(self, response):
        post_title_line_list = response.xpath("/html/body/table[2]/tr/td/table[2]/tr[1]/td/table/tr")
        for post_title_line in post_title_line_list:
            try:
                post_time_str = "".join(post_title_line.xpath("./td[6]//text()").extract()[:2]).replace("\xa0", "")
                post_title = post_title_line.xpath("./td[2]/font/span[@class='spnMessageText']/a/text()").extract()[0]
                if len(post_time_str) > 4:
                    post_time_stamp = get_time_stamp(post_time_str, None)
                    if time.time() * 1000 - post_time_stamp > 1 * 24 * 3600 * 1000:
                        logger.info("帖子《{}》超过1天没有更新，跳过".format(post_title))
                    else:
                        logger.info("帖子《{}》最近1天有更新，开始采集...".format(post_title))
                        post_uri = post_title_line.xpath(
                            "./td[2]/font/span[@class='spnMessageText']/a/@href").extract()[0]
                        request_meta = {
                            "current_page_num": 1
                        }
                        if self.proxy is not None:
                            request_meta["proxy"] = self.proxy
                        post_title_url = 'http://www.acewings.com/cobrachen/forum/' + post_uri
                        yield scrapy.Request(url=post_title_url,
                                             callback=self.parse,
                                             meta=request_meta)
            except:
                continue

    def parse(self, response, **kwargs):
        current_page_num = response.request.meta["current_page_num"]
        current_url = response.url
        post_area = response.xpath("/html/body/table[4]/tr/td/table/tr/td/table/tr")
        post_title = response.xpath("//title/text()").extract()[0].split("-")[1].strip()
        post_id = current_url.split("=")[1]
        if len(post_area) > 1:
            for line_index in range(1, len(post_area) - 1):
                current_line = post_area[line_index]
                author_cell = current_line.xpath("./td[1]")
                content_cell = current_line.xpath("./td[2]")
                post_time = "".join([_.strip() for _ in content_cell.xpath("./table//tr[1]/td//text()")
                                    .extract()]).replace("\xa0", " ")
                post_time = get_time_stamp(post_time,
                                           {r"(\d{2}/\d{2}/\d{4})\D*(\d{2}:\d{2}:\d{2})*\D*": ['%m-%d-%Y %H:%M:%S']})
                post_content_dict = {
                    "author_name": author_cell.xpath(".//span[@class='spnMessageText']/a/text()").extract()[0],
                    "author_link": author_cell.xpath(".//span[@class='spnMessageText']/a/@href").extract()[0],
                    "author_info": " ".join([_.strip() for _ in author_cell.xpath(".//text()").extract()]),
                    "post_time": post_time,
                    "post_content": "".join(content_cell.xpath("./table//tr[2]/td//text()").extract()),
                    "post_title": post_title,
                    "post_id": post_id
                }
                img_urls = content_cell.xpath(".//img/@src").extract()
                content_image_urls = []
                for u in img_urls:
                    if u.startswith("http"):
                        content_image_urls.append(u)
                mdc_tw_item = EsItem()
                mdc_tw_item['es_sid'] = get_str_md5(post_id + post_content_dict['post_content'])
                mdc_tw_item['es_urlname'] = current_url + '&floor={}'.format(line_index)
                mdc_tw_item['es_urltitle'] = post_content_dict['post_title']
                mdc_tw_item['es_lasttime'] = post_time
                mdc_tw_item['es_urltime'] = post_time
                mdc_tw_item['es_authors'] = post_content_dict['author_name']
                mdc_tw_item['es_urlimage'] = content_image_urls
                mdc_tw_item['es_urlcontent'] = post_content_dict['post_content']
                mdc_tw_item['es_doclength'] = len(post_content_dict['post_content'])
                mdc_tw_item['es_sitename'] = '台湾MDC第二论坛'
                mdc_tw_item['es_srcname'] = 'http://www.acewings.com/'
                mdc_tw_item['es_carriertype'] = 'forum'
                if current_page_num == 1 and line_index == 1:
                    # 页数、楼层都是1，说明是主贴
                    mdc_tw_item['es_isrepost'] = '0'
                    mdc_tw_item['es_hkey'] = post_content_dict['post_id']
                else:
                    mdc_tw_item['es_isrepost'] = '1'
                    mdc_tw_item['es_urltopic'] = post_content_dict['post_id']
                yield mdc_tw_item
        next_page_link = response.xpath("/html/body/table[3]//a[last()]/@href").extract()
        if len(next_page_link) > 0:
            next_page_link_address = buildUrl(current_url, [], next_page_link[0])
            next_page_num = current_page_num + 1
            request_meta = {
                "current_page_num": next_page_num
            }
            if self.proxy is not None:
                request_meta["proxy"] = self.proxy
            yield scrapy.Request(url=next_page_link_address,
                                 callback=self.parse,
                                 meta=request_meta)
