import json
import time

import scrapy

from CustomWebsite.items import WebsiteSpiderItem


class WebsiteSpider(scrapy.Spider):
    # 日本防卫省自卫队白皮书
    name = 'modjp_press_wp'

    def __init__(self, params=None, *args, **kwargs):
        super(WebsiteSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def start_requests(self):
        yield scrapy.Request(url='https://www.mod.go.jp/j/press/wp/index.html', callback=self.parse)

    def parse(self, response, **kwargs):
        page_title = response.xpath("//div[@class='content-section']/section/h2/text()").extract()[0]
        page_content = "<br>\r\n".join(
            response.xpath("//div[@class='publiction-content-text']/p[@class='paragraph-indent']").extract())
        current_timestamp = int(time.time() * 1000)
        pdf_links_info = response.xpath("//a[contains(@href,'.pdf')]")
        for line in pdf_links_info:
            pdf_name = line.xpath("./text()").extract()[0]
            pdf_uri = line.xpath("./@href").extract()[0]
            pdf_url = 'https://www.mod.go.jp/j/press/wp/' + pdf_uri
            page_item = WebsiteSpiderItem()
            page_item['es_urlname'] = pdf_url
            page_item['es_attachment'] = [pdf_url]
            # page_item['es_attention'] = pdf_name
            page_item['es_urlcontent'] = page_content
            page_item['es_urltitle'] = page_title
            page_item['es_urltime'] = current_timestamp
            page_item['es_lasttime'] = current_timestamp
            page_item['es_sitename'] = '日本防卫省-自卫队'
            page_item['es_srcname'] = 'https://www.mod.go.jp/'
            page_item['es_loadtime'] = str(int(time.time() * 1000))
            page_item['es_carriertype'] = 'article'
            page_item['es_catalog'] = '武装力量'
            page_item['es_catalog2'] = 'pdf'
            page_item['es_collection'] = '白皮书'
            page_item['es_lang'] = 'JPN'
            page_item['es_doclength'] = len(page_item['es_urlcontent'])
            yield page_item
