import json
import logging
import time

import scrapy
import validators

from PhotoSpiders.items import PhotoSpidersItem
from PhotoSpiders.utils.date_utils import get_time_stamp
from PhotoSpiders.utils.http_utils import build_url
from PhotoSpiders.utils.string_utils import has_tags_in_use, OTHERS_TAG


class WebsiteInfoDTSpider(scrapy.Spider):
    name = 'dt_images'

    def __init__(self, params=None, *args, **kwargs):
        super(WebsiteInfoDTSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def start_requests(self):
        for page in range(1, 20):
            yield scrapy.Request(f'https://www.defencetalk.com/military/photos/page-{page}',
                                 callback=self.parse_list_page)
            break

    def parse_list_page(self, response):
        detail_uris = response.xpath(
            "//div[@class='block-body']/div[@class='itemList']/div[contains(@class,'itemList-item')]/a/@href").extract()
        for uri in detail_uris:
            full_url = build_url(response.url, [], uri)
            if not validators.url(full_url):
                continue
            yield scrapy.Request(full_url, callback=self.parse)

    def parse(self, response, **kwargs):
        page_info = {
            'current_url': response.xpath("//meta[@property='og:url']/@content").extract()[0],
            'title': response.xpath("//meta[@property='og:title']/@content").extract()[0],
            'description': response.xpath("//meta[@property='og:description']/@content").extract()[0],
            'image_url': response.xpath(
                "//div[@class='media-container']//img[@class='js-mediaImage']/@src").extract()[0],
            'date_taken': response.xpath(
                "//div[@class='contentRow-lesser p-description']//time[@class='u-dt']/text()").extract()[0],
            'date_posted': response.xpath(
                "//div[@class='contentRow-lesser p-description']//time[@class='u-dt']/text()").extract()[0],
            'tags': response.xpath(
                "//span[@class='js-tagList']/a[@class='tagItem']//text()").extract()
        }
        logging.debug(page_info)
        page_item = PhotoSpidersItem()
        page_item['es_urlname'] = page_info['image_url']
        page_item['es_srcname'] = page_info['current_url']
        page_item['es_urltitle'] = page_info['title']
        page_item['es_urlcontent'] = page_info['description']
        page_item['es_urltime'] = get_time_stamp(page_info['date_taken'])
        page_item['es_lasttime'] = get_time_stamp(page_info['date_posted'])
        page_item['es_urlimage'] = ['https://www.defencetalk.com' + page_info['image_url']]
        page_item['es_sitename'] = 'Defence Talk'
        page_item['es_loadtime'] = str(int(time.time() * 1000))
        page_item['es_carriertype'] = 'news'
        page_item['es_doclength'] = len(page_item['es_urlcontent'])
        page_item['es_catalog2'] = []
        if not has_tags_in_use(page_info['tags']):
            page_info['tags'].append(OTHERS_TAG)
        page_item['es_tags'] = ';'.join(page_info['tags'])
        response.request.meta['CUSTOM_FILTER_KEY'] = 'URL_Filter:PhotoSpiders:DT_URL_Filter'
        response.request.meta['CUSTOM_FILTER_VAL'] = page_item['es_urlname']
        yield page_item
