import json
import logging
import time

import scrapy
import validators
from scrapy.loader import ItemLoader

from CustomWebsite.items import WebsiteSpiderItem
from CustomWebsite.utils.date_utils import get_time_stamp
from CustomWebsite.utils.http_utils import build_url


class WebsiteSpider(scrapy.Spider):
    name = 'air_mnd_news'

    start_urls = [
        'https://air.mnd.gov.tw/TW/News/News_List.aspx?CID=48',
        'https://air.mnd.gov.tw/TW/News/News_List.aspx?CID=213'
    ]

    def __init__(self, params=None, *args, **kwargs):
        super(WebsiteSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def parse(self, response, **kwargs):
        uris = response.xpath(
            "//main[@class='Content']//ul[@id='ContentPlaceHolder1_TableList_ul_List']/li/a/@href").extract()
        for uri in uris:
            full_url = build_url(response.url, [], uri)
            yield scrapy.Request(full_url, callback=self.parse_detail, meta={
                'CUSTOM_FILTER_KEY': 'URL_Filter:Custom:' + self.name,
                'CUSTOM_FILTER_VAL': full_url
            })

    def parse_detail(self, response):
        title = response.xpath("//h1//text()").extract()[0].replace('\n', '').replace(" ", "")
        content = "".join(response.xpath("//div[@class='Des']//p").extract())
        page_item = WebsiteSpiderItem()
        page_item['es_urltitle'] = title
        page_item['es_urlcontent'] = content
        time_str = response.xpath("//p[@class='Time']/text()").extract()[0].strip()
        page_item['es_urltime'] = get_time_stamp(time_str)
        page_item['es_urlname'] = response.url
        page_item['es_lasttime'] = page_item['es_urltime']
        page_item['es_sitename'] = '“中华民国空军”'
        page_item['es_srcname'] = 'https://air.mnd.gov.tw/'
        page_item['es_loadtime'] = str(int(time.time() * 1000))
        page_item['es_carriertype'] = 'news'
        page_item['es_catalog'] = '武装力量'
        page_item['es_collection'] = '新闻'
        page_item['es_lang'] = 'TWN'
        page_item['es_doclength'] = len(page_item['es_urlcontent'])
        # logging.info(page_item)
        response.request.meta['CUSTOM_FILTER_KEY'] = 'URL_Filter:Custom:' + self.name
        response.request.meta['CUSTOM_FILTER_VAL'] = page_item['es_urlname']
        yield page_item
