# -*- coding: utf-8 -*-
import json
import time

import scrapy

from CustomWebsite.items import WebsiteSpiderItem
from CustomWebsite.utils.date_utils import get_time_stamp


class ModJPApproachUpdateSpider(scrapy.Spider):
    name = 'modjp_approach_update'

    def __init__(self, params=None, *args, **kwargs):
        super(ModJPApproachUpdateSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def start_requests(self):
        yield scrapy.Request(url='https://www.mod.go.jp/j/approach/anpo/kyougi_add_list.js', callback=self.js_parse,
                             dont_filter=True)

    def js_parse(self, response):
        approach_js_start_index = response.text.index('[')
        approach_js_str = response.text[approach_js_start_index:]
        for keyword in ['year', 'date', 'title', 'url', 'cat', 'option']:
            approach_js_str = approach_js_str.replace(keyword, f'"{keyword}"')
        approach_js_str = f'{approach_js_str[:approach_js_str.rfind(",")]}]'
        approach_js_list = json.loads(approach_js_str)
        for js_info in approach_js_list:
            url_link_suffix = js_info['url']
            title = js_info['title']
            if url_link_suffix.endswith('.pdf'):
                pdf_link = f'https://www.mod.go.jp{url_link_suffix}'
                url_time_text = js_info['date']
                page_item = get_page_item(pdf_link, title, title, url_time_text, [pdf_link])
                response.request.meta['CUSTOM_FILTER_KEY'] = 'URL_Filter:Custom:' + self.name
                response.request.meta['CUSTOM_FILTER_VAL'] = page_item['es_urlname']
                yield page_item
                continue
            yield scrapy.Request(url=f'https://www.mod.go.jp{url_link_suffix}', callback=self.parse,
                                 meta={'js_info': js_info},
                                 dont_filter=True)

    def parse(self, response, **kwargs):
        js_info = response.request.meta['js_info']
        title_name = js_info['title']
        url_time_text = js_info['date']
        page_content = "<br>\r\n".join(
            response.xpath("//div[@class='content-section']//*[self::p or self::h2]").extract())
        file_link_suffix_list = response.xpath(
            "//div[@class='content-section']//*[self::p or self::h2]//a[contains(@href,'.pdf')]/@href").extract()
        link_url = response.url
        file_link_list = []
        if file_link_suffix_list is not None and len(file_link_suffix_list) > 0:
            for link in file_link_suffix_list:
                file_link_list.append(f'{link_url[:link_url.rfind("/")]}/{link}')
        page_item = get_page_item(link_url, page_content, title_name, url_time_text, file_link_list)
        response.request.meta['CUSTOM_FILTER_KEY'] = 'URL_Filter:Custom:' + self.name
        response.request.meta['CUSTOM_FILTER_VAL'] = page_item['es_urlname']
        yield page_item


def get_page_item(url_name, url_content, url_title, url_time_text, attachment):
    page_item = WebsiteSpiderItem()
    page_item['es_urlname'] = url_name
    page_item['es_urlcontent'] = url_content
    page_item['es_urltitle'] = url_title
    page_item['es_urltime'] = get_time_stamp(url_time_text)
    page_item['es_lasttime'] = get_time_stamp(url_time_text)
    page_item['es_loadtime'] = str(int(time.time() * 1000))
    page_item['es_attachment'] = attachment
    page_item['es_sitename'] = '日本防卫省-自卫队'
    page_item['es_srcname'] = 'https://www.mod.go.jp'
    page_item['es_carriertype'] = 'article'
    page_item['es_catalog'] = '武装力量'
    page_item['es_collection'] = '条令'
    page_item['es_lang'] = 'JPN'
    page_item['es_doclength'] = len(page_item['es_urlcontent'])
    return page_item
