import json
import logging
import time

import redis
import scrapy
from redisbloom.client import Client
from scrapy.utils.project import get_project_settings

from PhotoSpiders.items import PhotoSpidersItem


class ShipPhotosSpider(scrapy.Spider):
    name = 'ship_photos'
    custom_settings = {
        'ITEM_PIPELINES': {
            'PhotoSpiders.pipelines.ProtobufSavePipeline': 300,
            'PhotoSpiders.pipelines.ImageDownloadAndTagPipeline': 1
        },
        'DOWNLOAD_DELAY': 3,
    }
    settings = get_project_settings()
    bloom_filter = Client(host=settings['REDIS_HOST'],
                          port=settings['REDIS_PORT'],
                          password=settings['REDIS_PWD'])
    redis_client = redis.Redis(host=settings['REDIS_HOST'],
                               port=settings['REDIS_PORT'],
                               password=settings['REDIS_PWD'])

    def __init__(self, params=None, *args, **kwargs):
        super(ShipPhotosSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def start_requests(self):
        page_array = [1]
        # for i in range(8):
        #     page_array.append(random.randint(100, 960))
        for page_index in page_array:
            body = {
                "shipName": "",
                "shipNameSearchMode": "begins",
                "imo": "",
                "mmsi": "",
                "eni": "",
                "callSign": "",
                "category": "23",
                "includeSubCategories": "on",
                "flag": "",
                "homePort": "",
                "buildYear": "",
                "status": "",
                "classSociety": "",
                "builder": "",
                "owner": "",
                "manager": "",
                "resolution": "",
                "user": "",
                "country": "",
                "location": "",
                "sortBy": "newest",
                "perPage": 100,
                "page": page_index
            }
            yield scrapy.Request(url='https://www.shipspotting.com/ssapi/gallery-search', method="POST",
                                 headers={"Content-Type": "application/json"},
                                 body=json.dumps(body), callback=self.parse)

    def parse(self, response, **kwargs):
        rsp_text = response.text
        rsp_body = json.loads(rsp_text)
        if 'items' in rsp_body and len(rsp_body['items']) > 0:
            for rsp_item in rsp_body['items']:
                image_id = str(rsp_item['lid'])
                if self.bloom_filter.bfAdd("URL_Filter:PhotoSpiders:ShipPhotos_URL_Filter", image_id) <= 0:
                    logging.info(f"图片 {image_id} 已采集")
                else:
                    page_item = PhotoSpidersItem()
                    if len(str(rsp_item['mmsi_no'])) > 0:
                        page_item['es_hkey'] = rsp_item['mmsi_no']
                    elif rsp_item['mmsi_code']:
                        page_item['es_hkey'] = str(rsp_item['mmsi_code'])
                    page_item['es_urltime'] = rsp_item['photo_date'] * 1000
                    page_item['es_lasttime'] = rsp_item['date'] * 1000
                    page_item['es_heat'] = rsp_item['views']
                    page_item['es_likecount'] = rsp_item['likes']
                    page_item['es_abstract'] = rsp_item['description']
                    page_item['es_urltitle'] = rsp_item['title']
                    page_item['es_tags'] = rsp_item['title']
                    page_item['es_urlcontent'] = rsp_item['description']
                    page_item['es_pkey'] = rsp_item['imo_no']
                    page_item['es_district'] = rsp_item['port_portname'] + ' ' + rsp_item['port_mainport'] + ' ' + \
                                               rsp_item['port_country']
                    page_item['es_authors'] = rsp_item['real_name']
                    image_url = f'https://www.shipspotting.com/photos/middle/{image_id[-1]}/{image_id[-2]}/{image_id[-3]}/{image_id}.jpg'
                    page_item['es_urlname'] = image_url
                    page_item['es_urlimage'] = [image_url]
                    page_item['es_loadtime'] = str(int(time.time() * 1000))
                    page_item['es_sitename'] = 'shipspotting'
                    page_item['es_carriertype'] = 'image'
                    page_item['es_srcname'] = 'https://www.shipspotting.com/'
                    page_item['es_doclength'] = len(page_item['es_urlcontent'])
                    page_item['es_catalog'] = 'weapon'
                    page_item['es_catalog1'] = 'ship'
                    page_item['es_catalog2'] = []
                    yield page_item
