import json
import logging
import random
import time

import scrapy
import validators

from PhotoSpiders.items import PhotoSpidersItem
from PhotoSpiders.utils.date_utils import get_time_stamp
from PhotoSpiders.utils.http_utils import build_url
from PhotoSpiders.utils.string_utils import has_tags_in_use, OTHERS_TAG


class WebsiteInfoDimagesSpider(scrapy.Spider):
    name = 'dvids_images'

    custom_settings = {
        'ITEM_PIPELINES': {
            'PhotoSpiders.pipelines.ProtobufSavePipeline': 300,
            'PhotoSpiders.pipelines.ImageDownloadAndTagPipeline': 1
        }
    }

    def __init__(self, params=None, *args, **kwargs):
        super(WebsiteInfoDimagesSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def start_requests(self):
        page_num = [1, 2, 3, 4, 5, 6]
        for random_num in range(5):
            page_num.append(random.randint(1000, 50000))
        for page in page_num:
            yield scrapy.Request(f'https://www.dvidshub.net/search/?filter%5Btype%5D=image&page={page}',
                                 callback=self.parse_list_page)

    def parse_list_page(self, response):
        detail_uris = response.xpath("//a[contains(@class,'box-button')]/@href").extract()
        for uri in detail_uris:
            full_url = build_url(response.url, [], uri)
            if not validators.url(full_url):
                continue
            yield scrapy.Request(full_url, callback=self.parse, meta={
                'CUSTOM_FILTER_KEY': 'PhotoSpiders:DVIDS_URL_Filter',
                'CUSTOM_FILTER_VAL': full_url
            })

    def parse(self, response, **kwargs):
        page_info = {
            'current_url': response.url,
            'title': response.xpath("//meta[@property='og:title']/@content").extract()[0],
            'description': response.xpath("//meta[@property='og:description']/@content").extract()[0],
            'image_url': response.xpath("//meta[@property='og:image']/@content").extract()[0],
            'date_taken': response.xpath(
                "//table[contains(@class,'image_info_table')][1]//tr[1]/td[2]/text()").extract()[0],
            'date_posted': response.xpath(
                "//table[contains(@class,'image_info_table')][1]//tr[2]/td[2]/text()").extract()[0],
            'location': response.xpath(
                "//table[contains(@class,'image_info_table')][1]//tr[7]/td[2]/text()").extract()[0],
            'tags': response.xpath(
                "//div[@class='tag']/div[@class='readonly']/a/text()").extract()
        }
        logging.debug(page_info)
        page_item = PhotoSpidersItem()
        page_item['es_urlname'] = page_info['current_url']
        page_item['es_srcname'] = 'https://www.dvidshub.net/image'
        page_item['es_urltitle'] = page_info['title']
        page_item['es_urlcontent'] = page_info['description']
        page_item['es_urltime'] = get_time_stamp(page_info['date_taken'])
        page_item['es_lasttime'] = get_time_stamp(page_info['date_posted'])
        current_timestamp = 1000 * time.time()
        if page_item['es_urltime'] > current_timestamp:
            if page_item['es_lasttime'] < current_timestamp:
                page_item['es_urltime'] = page_item['es_lasttime']
            else:
                return
        page_item['es_urlimage'] = [page_info['image_url']]
        page_item['es_district'] = page_info['location']
        page_item['es_sitename'] = '国防视频图像分发系统网站'
        page_item['es_loadtime'] = str(int(time.time() * 1000))
        page_item['es_carriertype'] = 'image'
        page_item['es_doclength'] = len(page_item['es_urlcontent'])
        page_item['es_catalog2'] = []
        if not has_tags_in_use(page_info['tags']):
            page_info['tags'].append(OTHERS_TAG)
        page_item['es_tags'] = ';'.join(page_info['tags'])
        response.request.meta['CUSTOM_FILTER_KEY'] = 'URL_Filter:PhotoSpiders:DVIDS_URL_Filter'
        response.request.meta['CUSTOM_FILTER_VAL'] = page_item['es_urlname']
        yield page_item
