# -*- coding: utf-8 -*-
import json
import logging
import re
import time
import scrapy

from CustomWebsite.items import WebsiteSpiderItem
from CustomWebsite.utils.date_utils import get_time_stamp


class ModJPPressNewsSpider(scrapy.Spider):
    name = 'modjp_press_news'

    def __init__(self, params=None, *args, **kwargs):
        super(ModJPPressNewsSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def start_requests(self):
        yield scrapy.Request(url='https://www.mod.go.jp/j/press/news/news_ja.js', callback=self.js_parse,
                             dont_filter=True)

    def js_parse(self, response):
        news_ja_start_index = response.text.index('[')
        news_ja_str = response.text[news_ja_start_index:]
        news_ja_list = json.loads(news_ja_str)
        for js_info in news_ja_list:
            pdf_link_suffix = js_info['print']
            if pdf_link_suffix is None or len(pdf_link_suffix) == 0:
                continue
            yield scrapy.Request(url=f'https://www.mod.go.jp{js_info["link"]}', callback=self.parse,
                                 meta={'js_info': js_info},
                                 dont_filter=True)

    def parse(self, response, **kwargs):
        js_info = response.request.meta['js_info']
        page_content = "<br>\r\n".join(
            response.xpath("//div[@class='content-section']/section/*[position()=1 or position()>2]").extract())
        title_name = js_info['title']
        pdf_link_suffix = js_info['print']
        pdf_link = f'https://www.mod.go.jp{pdf_link_suffix}'
        month = js_info['mm']
        day = js_info['dd']
        url_link_suffix = js_info['link']
        match = re.search(r'/(\d{4})/', url_link_suffix)
        if match:
            url_time_text = f'{match.group(1)}/{month}/{day}'
        else:
            return
        page_item = WebsiteSpiderItem()
        page_item['es_urlname'] = response.url
        page_item['es_urlcontent'] = page_content
        page_item['es_urltitle'] = title_name
        page_item['es_urltime'] = get_time_stamp(url_time_text)
        page_item['es_lasttime'] = get_time_stamp(url_time_text)
        page_item['es_loadtime'] = str(int(time.time() * 1000))
        page_item['es_attachment'] = [pdf_link]
        page_item['es_sitename'] = '日本防卫省-自卫队'
        page_item['es_srcname'] = 'https://www.mod.go.jp'
        page_item['es_carriertype'] = 'news'
        page_item['es_catalog'] = '武装力量'
        page_item['es_lang'] = 'JPN'
        page_item['es_collection'] = '条令' if '/approach/' in pdf_link_suffix else '新闻'
        page_item['es_doclength'] = len(page_item['es_urlcontent'])
        response.request.meta['CUSTOM_FILTER_KEY'] = 'URL_Filter:Custom:' + self.name
        response.request.meta['CUSTOM_FILTER_VAL'] = page_item['es_urlname']
        yield page_item
