# -*- coding: utf-8 -*-
import scrapy
import time
import json
import redis
from AirplaneSpiders.items import FlightBaseItem


# 航班信息采集--数据源：flightradar24 #
class FlightBase2Spider(scrapy.Spider):
    name = 'flightradar24_flight_base'
    allowed_domains = ['flightradar24.com']
    # start_urls = ['http://flightradar24.com/']
    flightradar24_api = 'https://data-live.flightradar24.com/clickhandler/?version=1.5&flight='

    custom_settings = {
        'DOWNLOAD_DELAY': 0.2,
        'PROTO_MODULE_PATH': 'AirplaneSpiders.proto.Plane_pb2',
        'PROTO_CLASS_NAME': 'FligthInfoSets',
        'PROTO_FIELD_NAME': 'FligthInfo',
        'PROTO_SAVE_FILE_PATH': r'/usr/local/spider_data',
        'PROTO_SAVE_FILE_NAME': r'track_data_flight_',
        'BATCH_SAVE_SIZE': 100,
        'ITEM_PIPELINES': {
            'AirplaneSpiders.pipelines.ProtobufSavePipeline': 300
        }
    }

    crawledRegs = []

    def __init__(self, params=None, *args, **kwargs):
        super(FlightBase2Spider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def start_requests(self):
        self.redis_client = redis.Redis(host=self.settings['REDIS_HOST'], port=int(self.settings['REDIS_PORT']),
                                        password=self.settings['REDIS_PWD'])
        self.aRegs = self.redis_client.smembers('AirplaneSpiders:aRegs_2')  # 航班id（flightradar24内部id）
        self.aRegList = []
        for aReg in self.aRegs:
            self.aRegList.append(aReg.decode('utf-8'))
            self.redis_client.srem('AirplaneSpiders:aRegs_2', aReg.decode('utf-8'))
        # 获取最近五十分钟内采集过航班号
        aRegsCrawled = self.redis_client.smembers('AirplaneSpiders:aRegsCrawled_2')
        aRegListCrawled = []
        for aReg in aRegsCrawled:
            aRegObj = json.loads(aReg.decode('utf-8'))
            if time.time() - aRegObj['expire'] > 0:
                self.redis_client.srem('AirplaneSpiders:aRegsCrawled_2', aReg.decode('utf-8'))
            else:
                aRegListCrawled.append(aRegObj['areg'])
        self.redis_client.close()
        for a in self.aRegList:
            aFlightObj = json.loads(a)
            if aFlightObj['areg'] in aRegListCrawled:
                continue
            yield scrapy.Request(self.flightradar24_api + aFlightObj['flightid'], callback=self.flight_base_parse)

    def flight_base_parse(self, response):
        item = FlightBaseItem()
        try:
            data = json.loads(response.text)
            item['FlightID'] = '2#%s' % data['identification']['id']
            if data['airline'] is not None and data['airline']['code'] is not None:
                item['ICAO'] = data['airline']['code']['icao']
                item['IATA'] = data['airline']['code']['iata']
            item['PlaneReg'] = data['aircraft']['registration']
            if data['airport'] is not None and data['airport']['origin'] is not None:
                item['TakeoffBase'] = data['airport']['origin']['name']
                item['TakeoffBaseICAO'] = data['airport']['origin']['code']['icao']
                item['TakeoffBaseIATA'] = data['airport']['origin']['code']['iata']
            if data['airport'] is not None and data['airport']['destination'] is not None:
                item['LandBase'] = data['airport']['destination']['name']
                item['LandBaseICAO'] = data['airport']['destination']['code']['icao']
                item['LandBaseIATA'] = data['airport']['destination']['code']['iata']
            if data['time'] is not None and data['time']['scheduled'] is not None:
                if data['time']['scheduled']['departure'] is not None and data['time']['scheduled'][
                    'departure'] != 'None':
                    item['ScheduleBegTime'] = str(data['time']['scheduled']['departure']) + '000'
                if data['time']['scheduled']['arrival'] is not None and data['time']['scheduled']['arrival'] != 'None':
                    item['ScheduleArrTime'] = str(data['time']['scheduled']['arrival']) + '000'
            if data['time'] is not None and data['time']['real'] is not None:
                if data['time']['real']['departure'] is not None and data['time']['real']['departure'] != 'None':
                    item['ActualBegTime'] = str(data['time']['real']['departure']) + '000'
                if data['time']['real']['arrival'] is not None and data['time']['real']['arrival'] != 'None':
                    item['ActualArrTime'] = str(data['time']['real']['arrival']) + '000'
            item['UpdateTime'] = str(int(time.time() * 1000))
            item['LastTime'] = str(data['firstTimestamp']) + '000'

            # 将已采集航班的飞机注册号存到已采集集合中，并设置过期时间五十分钟
            expire = time.time() + 50 * 60
            self.crawledRegs.append({'areg': item['PlaneReg'], 'expire': expire})
            yield item
        except Exception as e:
            self.logger.error('来自数据源 flightradar24 的航班基础信息解析错误，原因是：', e)

    def closed(self, reason):
        self.redis_client = redis.Redis(host=self.settings['REDIS_HOST'], port=int(self.settings['REDIS_PORT']),
                                        password=self.settings['REDIS_PWD'])
        for obj in self.crawledRegs:
            self.redis_client.sadd('AirplaneSpiders:aRegsCrawled_2', json.dumps(obj))
        self.redis_client.close()
