import json
import logging
import time

import scrapy
import validators
from scrapy.loader import ItemLoader

from CustomWebsite.items import WebsiteSpiderItem
from CustomWebsite.utils.date_utils import get_time_stamp
from CustomWebsite.utils.http_utils import build_url


class WebsiteSpider(scrapy.Spider):
    name = 'modjp_policy_common'

    start_urls = [
        'https://www.mod.go.jp/j/policy/agenda/guideline/index.html',
        'https://www.mod.go.jp/j/policy/defense/bmd/index.html',
        'https://www.mod.go.jp/j/policy/defense/technology_guideline/index.html'
    ]

    def __init__(self, params=None, *args, **kwargs):
        super(WebsiteSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def parse(self, response, **kwargs):
        page_item = WebsiteSpiderItem()
        item_loader = ItemLoader(item=page_item, response=response)
        item_loader.add_xpath('es_urltitle', "//div[@class='c-inner']/h1/text()")
        item_loader.add_xpath('es_urlcontent', "/html/body/div[@class='c-inner']//p")
        item_loader.add_xpath('es_urltime', "//div[@class='content-section']/p[last()]/text()")
        item_loader.add_xpath('es_attachment', "//a[contains(@href,'.pdf')]/@href")
        page_item = item_loader.load_item()
        pdf_urls = []
        for uri in page_item['es_attachment']:
            pdf_url = build_url(response.url, [], uri)
            if validators.url(pdf_url):
                pdf_urls.append(pdf_url)
        page_item['es_attachment'] = pdf_urls
        page_item['es_urltitle'] = ''.join(page_item['es_urltitle'])
        page_item['es_urlcontent'] = ''.join(page_item['es_urlcontent'])
        page_item['es_urltime'] = get_time_stamp(''.join(page_item['es_urltime']))
        page_item['es_urlname'] = response.url
        page_item['es_lasttime'] = page_item['es_urltime']
        page_item['es_attention'] = page_item['es_urltitle']
        page_item['es_sitename'] = '日本防卫省-自卫队'
        page_item['es_srcname'] = 'https://www.mod.go.jp/'
        page_item['es_loadtime'] = str(int(time.time() * 1000))
        page_item['es_carriertype'] = 'article'
        page_item['es_catalog'] = '武装力量'
        page_item['es_catalog2'] = 'pdf'
        page_item['es_collection'] = '政策'
        page_item['es_lang'] = 'JPN'
        page_item['es_doclength'] = len(page_item['es_urlcontent'])
        response.request.meta['CUSTOM_FILTER_KEY'] = 'URL_Filter:Custom:' + self.name
        response.request.meta['CUSTOM_FILTER_VAL'] = page_item['es_urlname']
        yield page_item
