import json
import time

import scrapy
import validators

from CustomWebsite.items import WebsiteSpiderItem
from CustomWebsite.utils.date_utils import get_time_stamp
from CustomWebsite.utils.http_utils import build_url
from CustomWebsite.utils.string_utils import has_tags_in_use, OTHERS_TAG


class WebsiteSpider(scrapy.Spider):
    name = 'dvids_news'
    custom_settings = {
        'ITEM_PIPELINES': {
            'CustomWebsite.pipelines.ProtobufSavePipeline': 300,
            'CustomWebsite.pipelines.ImageDownloadAndTagPipeline': 1
        }
    }

    def __init__(self, params=None, *args, **kwargs):
        super(WebsiteSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def start_requests(self):
        for page in range(1, 200):
            yield scrapy.Request(f'https://www.dvidshub.net/search/?filter%5Btype%5D=news&page={page}',
                                 callback=self.parse_list_page)
            break

    def parse_list_page(self, response):
        detail_uris = response.xpath("//a[contains(@class,'box-button')]/@href").extract()
        for uri in detail_uris:
            full_url = build_url(response.url, [], uri)
            if not validators.url(full_url):
                continue
            yield scrapy.Request(full_url, callback=self.parse, meta={
                'CUSTOM_FILTER_KEY': 'URL_Filter:Custom:' + self.name,
                'CUSTOM_FILTER_VAL': full_url
            })

    def parse(self, response, **kwargs):
        page_item = WebsiteSpiderItem()
        raw_paragraph = response.xpath("//div[@class='uk-grid'][2]/div[contains(@class,'news')]/p").extract()
        page_item['es_urlname'] = response.url
        page_item['es_urltitle'] = response.xpath("//h1[@class='asset-title']//text()").extract()[0]
        page_item['es_urlcontent'] = ''.join(raw_paragraph)
        page_item['es_urltime'] = get_time_stamp(
            response.xpath("//div[@class='uk-grid asset_information']//h3[2]/text()").extract()[0])
        page_item['es_lasttime'] = page_item['es_urltime']
        current_timestamp = 1000 * time.time()
        if page_item['es_urltime'] > current_timestamp:
            return
        page_item['es_urlimage'] = response.xpath("//div[@class='relatedimage']//img/@src").extract()
        tags = response.xpath("//div[@class='tag']/div[@class='readonly']/a/text()").extract()
        if not has_tags_in_use(tags):
            tags.append(OTHERS_TAG)
        page_item['es_tags'] = ';'.join(tags)
        page_item['es_sitename'] = '国防视频图像分发系统网站'
        page_item['es_srcname'] = 'https://www.dvidshub.net/news'
        page_item['es_loadtime'] = str(int(time.time() * 1000))
        page_item['es_carriertype'] = 'news'
        page_item['es_catalog'] = '武装力量'
        page_item['es_collection'] = '新闻'
        page_item['es_lang'] = 'USA'
        page_item['es_doclength'] = len(page_item['es_urlcontent'])
        response.request.meta['CUSTOM_FILTER_KEY'] = 'URL_Filter:Custom:' + self.name
        response.request.meta['CUSTOM_FILTER_VAL'] = page_item['es_urlname']
        yield page_item
