import json
import logging
import time

import scrapy
import validators
from scrapy.loader import ItemLoader

from CustomWebsite.items import WebsiteSpiderItem
from CustomWebsite.utils.date_utils import get_time_stamp
from CustomWebsite.utils.http_utils import build_url


class WebsiteSpider(scrapy.Spider):
    name = 'mac_tw_nb'

    start_urls = [
        'https://www.mac.gov.tw/cp.aspx?n=1F20DA5F1FF10573',
        'https://www.mac.gov.tw/cp.aspx?n=62AB3DD0858BEDBA',
        'https://www.mac.gov.tw/cp.aspx?n=2CB626AD417473AC',
        'https://www.mac.gov.tw/cp.aspx?n=DC71D80F35AD4FA4'
    ]

    def __init__(self, params=None, *args, **kwargs):
        super(WebsiteSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def parse(self, response, **kwargs):
        content = "".join(response.xpath("//div[@class='area-editor user-edit']//a[@class='pdf']/text()").extract())
        pdf_url = response.xpath("//div[@class='area-editor user-edit']//a[@class='pdf']/@href").extract()[0]
        page_item = WebsiteSpiderItem()
        page_item['es_attachment'] = [pdf_url]
        page_item['es_urltitle'] = content
        page_item['es_urlcontent'] = content
        try:
            time_str = response.xpath("//div[@class='list-text detail']//span/text()").extract()[0].strip()
            page_item['es_urltime'] = get_time_stamp(time_str)
            if page_item['es_urltime'] == 0:
                page_item['es_urltime'] = int(time.time() * 1000)
        except:
            page_item['es_urltime'] = int(time.time() * 1000)
        page_item['es_urlname'] = pdf_url
        page_item['es_lasttime'] = page_item['es_urltime']
        page_item['es_attention'] = page_item['es_urltitle']
        page_item['es_sitename'] = '台湾“行政院”大陆委员会'
        page_item['es_srcname'] = 'https://www.mac.gov.tw/'
        page_item['es_loadtime'] = str(int(time.time() * 1000))
        page_item['es_carriertype'] = 'article'
        page_item['es_catalog'] = '政府部门'
        page_item['es_catalog2'] = 'pdf'
        page_item['es_collection'] = '白皮书'
        page_item['es_lang'] = 'TWN'
        page_item['es_doclength'] = len(page_item['es_urlcontent'])
        # logging.info((page_item['es_attachment'], page_item['es_attention'], page_item['es_urltime']))
        response.request.meta['CUSTOM_FILTER_KEY'] = 'URL_Filter:Custom:' + self.name
        response.request.meta['CUSTOM_FILTER_VAL'] = page_item['es_urlname']
        yield page_item
