import json
import logging
import time

import cloudscraper
import scrapy

from CustomWebsite.items import WebsiteSpiderItem
from CustomWebsite.utils.date_utils import get_time_stamp


class WebsiteSpider(scrapy.Spider):
    name = 'congress_reports'
    start_urls = ['https://www.google.com']

    def __init__(self, params=None, *args, **kwargs):
        super(WebsiteSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def parse(self, response, **kwargs):
        url = "https://crsreports.congress.gov/search/results?term=&orderBy=Date&"
        scraper = cloudscraper.create_scraper()
        rsp_text = scraper.get(url).text
        rsp_body = json.loads(rsp_text)
        results = rsp_body['SearchResults']
        for line in results:
            title = line['Title']
            product_number = line['ProductNumber']
            product_type_code = line['ProductTypeCode']
            pdf_url = f'https://crsreports.congress.gov/product/pdf/download/{product_type_code}/{product_number}/{product_number}.pdf/'
            page_item = WebsiteSpiderItem()
            page_item['es_attachment'] = [pdf_url]
            page_item['es_urltitle'] = title
            page_item['es_urlcontent'] = title
            page_item['es_urltime'] = get_time_stamp(line['CoverDate'])
            page_item['es_urlname'] = pdf_url
            page_item['es_lasttime'] = page_item['es_urltime']
            page_item['es_attention'] = page_item['es_urltitle']
            page_item['es_sitename'] = '美国国会研究处'
            page_item['es_srcname'] = 'https://crsreports.congress.gov/'
            page_item['es_loadtime'] = str(int(time.time() * 1000))
            page_item['es_carriertype'] = 'article'
            page_item['es_catalog'] = '政府部门'
            page_item['es_catalog2'] = 'pdf'
            page_item['es_collection'] = '白皮书'
            page_item['es_lang'] = 'USA'
            page_item['es_doclength'] = len(page_item['es_urlcontent'])
            # logging.info((page_item['es_attachment'], page_item['es_attention'], page_item['es_urltime']))
            response.request.meta['CUSTOM_FILTER_KEY'] = 'URL_Filter:Custom:' + self.name
            response.request.meta['CUSTOM_FILTER_VAL'] = page_item['es_urlname']
            yield page_item
