import json
import time

import scrapy
import validators

from CustomWebsite.items import WebsiteSpiderItem
from CustomWebsite.utils.date_utils import get_time_stamp
from CustomWebsite.utils.http_utils import build_url, filter_html_tags


class WebsiteSpider(scrapy.Spider):
    name = 'rand_report'

    def __init__(self, params=None, *args, **kwargs):
        super(WebsiteSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def start_requests(self):
        yield scrapy.Request('https://www.rand.org/pubs.html', callback=self.parse_list_page)
        # for index in range(10):
        #     page = index + 1
        #     yield scrapy.Request(f'https://www.rand.org/pubs.html?pub-date=20220801%3A&page={page}',
        #                          callback=self.parse_list_page)

    def parse_list_page(self, response):
        detail_uris = response.xpath(
            "//div[@class='filteredlist-data']//li[contains(@class,'pagination')=0]/a/@href").extract()
        for uri in detail_uris:
            full_url = build_url(response.url, [], uri)
            yield scrapy.Request(full_url, callback=self.parse, meta={
                'CUSTOM_FILTER_KEY': 'URL_Filter:Custom:' + self.name,
                'CUSTOM_FILTER_VAL': full_url
            })

    def parse(self, response, **kwargs):
        page_item = WebsiteSpiderItem()
        raw_paragraph = response.xpath("//meta[@name='description']/@content").extract()
        page_item['es_urlname'] = response.url
        page_item['es_urltitle'] = response.xpath("//meta[@name='title-terms']/@content").extract()[0]
        page_item['es_urlcontent'] = filter_html_tags(''.join(raw_paragraph))
        page_item['es_urltime'] = get_time_stamp(
            response.xpath("//meta[@name='citation_online_date']/@content").extract()[0])
        page_item['es_lasttime'] = page_item['es_urltime']
        page_item['es_sitename'] = '美国兰德公司'
        page_item['es_srcname'] = 'https://www.rand.org/pubs.html'
        page_item['es_loadtime'] = str(int(time.time() * 1000))
        page_item['es_carriertype'] = 'article'
        page_item['es_doclength'] = len(page_item['es_urlcontent'])
        page_item['es_catalog'] = '安全智库'
        page_item['es_collection'] = '时评'
        page_item['es_lang'] = 'USA'
        file_urls = []
        page_pdf_links = response.xpath(
            "//div[@class='product-header-cols']//div[@class='btn-download']/a/@href").extract()
        for uri in page_pdf_links:
            pdf_url = build_url(response.url, [], uri)
            if not validators.url(pdf_url):
                continue
            file_urls.append(pdf_url)
        if len(file_urls) > 0:
            page_item['es_attachment'] = file_urls
            response.request.meta['CUSTOM_FILTER_KEY'] = 'URL_Filter:Custom:' + self.name
            response.request.meta['CUSTOM_FILTER_VAL'] = page_item['es_urlname']
            yield page_item
