import json
import logging
import time

import scrapy
import validators
import random

from PhotoSpiders.items import PhotoSpidersItem
from PhotoSpiders.utils.date_utils import get_time_stamp
from PhotoSpiders.utils.http_utils import build_url


class WebsiteInfoJetphotosImagesSpider(scrapy.Spider):
    name = 'jetphotos_images'
    custom_settings = {
        'ITEM_PIPELINES': {
            'PhotoSpiders.pipelines.ProtobufSavePipeline': 300,
            'PhotoSpiders.pipelines.ImageDownloadAndTagPipeline': 1
        },
        'DOWNLOAD_DELAY': 4,
    }

    def __init__(self, params=None, *args, **kwargs):
        super(WebsiteInfoJetphotosImagesSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def start_requests(self):
        page_array = [1]
        for i in range(8):
            page_array.append(random.randint(100, 8105))
        for page in page_array:
            yield scrapy.Request(f'https://www.jetphotos.com/showphotos.php?genre=2&sort-order=0&page={page}',
                                 callback=self.parse_list_page)

    def parse_list_page(self, response):
        detail_uris = response.xpath("//div[@class='result']//a[@class='result__photoLink']/@href").extract()
        for uri in detail_uris:
            full_url = build_url(response.url, [], uri)
            if not validators.url(full_url):
                continue
            yield scrapy.Request(full_url, callback=self.parse, meta={
                'CUSTOM_FILTER_KEY': 'PhotoSpiders:Jetphotos_URL_Filter',
                'CUSTOM_FILTER_VAL': full_url
            })

    def parse(self, response, **kwargs):
        logging.debug("start parsing")
        page_info = {
            'current_url': response.xpath("//meta[@property='og:url']/@content").extract()[0],  # 网页 URL
            'Registration': response.xpath("//div[@class='grid']/div[@class='col-4'][1]//h4/text()").extract()[0],
            'Photo Date': response.xpath("//div[@class='grid']/div/div[@class='col'][1]/h4/text()").extract()[0],
            'Uploaded': response.xpath("//div[@class='grid']/div/div[@class='col'][2]/h4/text()").extract()[0],
            'Views': response.xpath(
                "//div[@class='grid']/div[@class='col-4'][2]/div/div[@class='col']//h4/text()").extract()[0],
            'Likes': response.xpath(
                "//div[@class='grid']/div[@class='col-4'][2]//div[@class='col photoLikes']/h4/text()").extract()[0],
            'Badges': ''.join(
                response.xpath("//div[@class='grid-noBottom']//div[@class='badges']//img/@alt").extract()),
            'Notes': ''.join(response.xpath(
                "//div[@class='wrapper__content wrapper__content--small'][1]/div[@class='grid-noBottom']/div[@class='col-4'][2]/span/text()").extract()),
            'Camera': ''.join([_.strip().replace('\t', '') for _ in response.xpath(
                "//div[@class='wrapper__content wrapper__content--small'][1]/div[@class='grid-noBottom']/div[@class='col-4'][3]/span/text()").extract()]),
            'Aircraft':
                response.xpath("//div[@class='grid-noBottom']//li[@class='list__item'][3]//h2//text()").extract()[0],
            'Airline':
                response.xpath("//div[@class='grid-noBottom']//li[@class='list__item'][4]/span/a/text()").extract()[0],
            'Serial':
                response.xpath("//div[@class='grid-noBottom']//li[@class='list__item'][5]//h4/a/text()").extract()[
                    0].strip(),
            'Location': ', '.join(response.xpath(
                "//div[@class='grid-noBottom']/div[@class='col-4'][2]//li[@class='list__item'][position()>1]//a/text()").extract()),
            'Author':
                response.xpath("//div[@class='grid-noBottom']//li[@class='list__item'][2]/span/h6/text()").extract()[0],
            'photo_url': response.xpath(
                "//div[@class='large-photo__desktop']/a[@id='show-large-photo']/img[@class='large-photo__img']/@src").extract()[
                0]
        }
        logging.debug(page_info)
        page_item = PhotoSpidersItem()
        page_item['es_hkey'] = page_info['Registration']
        page_item['es_urltime'] = get_time_stamp(page_info['Photo Date'])
        page_item['es_lasttime'] = get_time_stamp(page_info['Uploaded'])
        page_item['es_heat'] = page_info['Views']
        page_item['es_likecount'] = page_info['Likes']
        page_item['es_positiveWords'] = page_info['Badges']
        page_item['es_abstract'] = page_info['Notes']
        page_item['es_reportinfo'] = page_info['Camera']
        page_item['es_urltitle'] = page_info['Aircraft']
        page_item['es_tags'] = page_info['Aircraft']
        page_item['es_urlcontent'] = page_info['Airline']
        page_item['es_pkey'] = page_info['Serial']
        page_item['es_district'] = page_info['Location']
        page_item['es_authors'] = page_info['Author']
        page_item['es_urlimage'] = [page_info['photo_url']]
        page_item['es_loadtime'] = str(int(time.time() * 1000))
        page_item['es_urlname'] = page_info['current_url']
        page_item['es_sitename'] = 'JETPHOTOS'
        page_item['es_carriertype'] = 'image'
        page_item['es_srcname'] = 'https://www.jetphotos.com/'
        page_item['es_doclength'] = len(page_item['es_urlcontent'])
        page_item['es_catalog'] = 'weapon'
        page_item['es_catalog1'] = 'plane'
        page_item['es_catalog2'] = []
        response.request.meta['CUSTOM_FILTER_KEY'] = 'URL_Filter:PhotoSpiders:Jetphotos_URL_Filter'
        response.request.meta['CUSTOM_FILTER_VAL'] = page_item['es_urlname']
        yield page_item
