import json
import logging
import time

import scrapy
import validators
from scrapy.loader import ItemLoader

from CustomWebsite.items import WebsiteSpiderItem
from CustomWebsite.utils.date_utils import get_time_stamp
from CustomWebsite.utils.http_utils import build_url


class WebsiteSpider(scrapy.Spider):
    name = 'modjp_presiding_houan'

    start_urls = [
        'https://www.mod.go.jp/j/presiding/houan/index.html',
        'https://www.mod.go.jp/j/presiding/houan/houan_192-200.html'
    ]

    def __init__(self, params=None, *args, **kwargs):
        super(WebsiteSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def parse(self, response, **kwargs):
        table_area = response.xpath("//div[@class='content-section']/section/table")
        for table in table_area:
            time_str = table.xpath(".//tr[1]/td/text()").extract()[0].strip()
            content = table.xpath(".//tr[2]/td/text()").extract()[0].strip()
            title = content[:75]
            pdf_lines = table.xpath(".//tr[3]/td//a")
            for line in pdf_lines:
                pdf_name = line.xpath("./text()").extract()[0].replace('\n', '').replace(" ", "").replace("\t", "")
                pdf_name = title + ' - ' + pdf_name
                pdf_url = line.xpath("./@href").extract()[0]
                pdf_url = build_url(response.url, [], pdf_url)
                page_item = WebsiteSpiderItem()
                page_item['es_attachment'] = [pdf_url]
                page_item['es_urltitle'] = title
                page_item['es_urlcontent'] = content
                page_item['es_urltime'] = get_time_stamp(time_str)
                page_item['es_urlname'] = pdf_url
                page_item['es_lasttime'] = page_item['es_urltime']
                page_item['es_attention'] = pdf_name
                page_item['es_sitename'] = '日本防卫省-自卫队'
                page_item['es_srcname'] = 'https://www.mod.go.jp/'
                page_item['es_loadtime'] = str(int(time.time() * 1000))
                page_item['es_carriertype'] = 'article'
                page_item['es_catalog'] = '武装力量'
                page_item['es_catalog2'] = 'pdf'
                page_item['es_collection'] = '法律'
                page_item['es_lang'] = 'JPN'
                page_item['es_doclength'] = len(page_item['es_urlcontent'])
                # logging.info((page_item['es_attachment'], page_item['es_attention'], page_item['es_urltime']))
                response.request.meta['CUSTOM_FILTER_KEY'] = 'URL_Filter:Custom:' + self.name
                response.request.meta['CUSTOM_FILTER_VAL'] = page_item['es_urlname']
                yield page_item
