# -*- coding: utf-8 -*-
import json
import logging
import re
import time

import scrapy

from CustomWebsite.items import WebsiteSpiderItem
from CustomWebsite.utils.date_utils import get_time_stamp


class ModJPPressUpdateSpider(scrapy.Spider):
    name = 'modjp_press_update'

    def __init__(self, params=None, *args, **kwargs):
        super(ModJPPressUpdateSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def start_requests(self):
        yield scrapy.Request(url='https://www.mod.go.jp/j/update/update_ja.js', callback=self.js_parse,
                             dont_filter=True)

    def js_parse(self, response):
        update_ja_start_index = response.text.index('[')
        update_ja_str = response.text[update_ja_start_index:]
        update_ja_list = json.loads(update_ja_str)
        for js_info in update_ja_list:
            url_link_suffix = js_info['link']
            if '/press/' not in url_link_suffix:
                continue
            if url_link_suffix.endswith('index.html'):
                continue
            yield scrapy.Request(url=f'https://www.mod.go.jp{js_info["link"]}', callback=self.parse,
                                 meta={'js_info': js_info},
                                 dont_filter=True)

    def parse(self, response, **kwargs):
        js_info = response.request.meta['js_info']
        page_content = "<br>\r\n".join(
            response.xpath("//div[@class='content-section']/*").extract())
        title_name = js_info['title']
        file_link_suffix_list = response.xpath(
            "//div[@class='content-section']//a[contains(@href,'.pdf') or contains(@href,'.xlsx')]/@href").extract()
        link_url = response.url
        file_link_list = []
        if file_link_suffix_list is not None and len(file_link_suffix_list) > 0:
            for link in file_link_suffix_list:
                file_link_list.append(f'{link_url[:link_url.rfind("/")]}{link}')
        month = js_info['mm']
        day = js_info['dd']
        match = re.search(r'/(\d{4})/', link_url)
        if match:
            url_time_text = f'{match.group(1)}/{month}/{day}'
        else:
            return
        page_item = WebsiteSpiderItem()
        page_item['es_urlname'] = link_url
        page_item['es_urlcontent'] = page_content
        page_item['es_urltitle'] = title_name
        page_item['es_urltime'] = get_time_stamp(url_time_text)
        page_item['es_lasttime'] = get_time_stamp(url_time_text)
        page_item['es_loadtime'] = str(int(time.time() * 1000))
        page_item['es_attachment'] = file_link_list
        page_item['es_sitename'] = '日本防卫省-自卫队'
        page_item['es_srcname'] = 'https://www.mod.go.jp'
        page_item['es_carriertype'] = 'article'
        page_item['es_catalog'] = '武装力量'
        page_item['es_collection'] = '新闻'
        page_item['es_lang'] = 'JPN'
        page_item['es_doclength'] = len(page_item['es_urlcontent'])
        # print(page_item)
        response.request.meta['CUSTOM_FILTER_KEY'] = 'URL_Filter:Custom:' + self.name
        response.request.meta['CUSTOM_FILTER_VAL'] = page_item['es_urlname']
        yield page_item
