# -*- coding: utf-8 -*-
import json
import re
import time
import redis
import scrapy
from AirplaneSpiders.items import FlightBaseItem


# 非常准航班基本信息采集#
class FlightBaseSpider(scrapy.Spider):
    name = 'radarbox_flight_base'
    allowed_domains = ['adsbapi.variflight.com']
    base_api = 'https://adsbapi.variflight.com/adsb/index/flightdetail?lang=zh_CN&'
    radarbox_live_api = 'https://data.rb24.com/live-flight-info?fid='
    # radarbox_api = 'https://www.radarbox.com/data/get-replay?fid='
    radarbox_history_api = 'https://www.radarbox.com/search/advanced?q='
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'CONCURRENT_REQUESTS_PER_DOMAIN': 4,
        'BATCH_SAVE_SIZE': 100,
        'PROTO_MODULE_PATH': 'AirplaneSpiders.proto.Plane_pb2',
        'PROTO_CLASS_NAME': 'FligthInfoSets',
        'PROTO_FIELD_NAME': 'FligthInfo',
        'PROTO_SAVE_FILE_PATH': r'/usr/local/spider_data',
        'PROTO_SAVE_FILE_NAME': r'track_data_flight_',
        'ITEM_PIPELINES': {
            'AirplaneSpiders.pipelines.ProtobufSavePipeline': 300
        }
    }
    crawledRegs = []

    def __init__(self, params=None, *args, **kwargs):
        super(FlightBaseSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def start_requests(self):
        self.redis_client = redis.Redis(host=self.settings['REDIS_HOST'], port=int(self.settings['REDIS_PORT']),
                                        password=self.settings['REDIS_PWD'])
        self.aRegs = self.redis_client.smembers('AirplaneSpiders:aRegs')  # 飞机注册号和航班id（该id为radarbox内部id）
        self.aRegList = []
        for aReg in self.aRegs:
            self.aRegList.append(aReg.decode('utf-8'))
            self.redis_client.srem('AirplaneSpiders:aRegs', aReg.decode('utf-8'))
        # 获取最近240分钟内采集过航班的飞机注册号
        fid_crawled = self.redis_client.smembers('AirplaneSpiders:fidCrawled')
        fid_crawled_list = []
        for fid in fid_crawled:
            fid = json.loads(fid.decode('utf-8'))
            if time.time() - fid['expire'] > 0:
                self.redis_client.srem('AirplaneSpiders:fidCrawled', json.dumps(fid))
            else:
                fid_crawled_list.append(fid['fid'])

        for aReg in self.aRegList:
            try:
                aRegObj = json.loads(aReg)
                if aRegObj['flightid'] in fid_crawled_list:
                    continue
                # yield scrapy.Request(self.base_api+ 'anum=%s&onground=0'%aRegObj['areg'].replace('-',''), callback = self.flight_base_parse, meta={'areg':aRegObj})
                # yield SplashRequest(url = self.radarbox_api+ aRegObj['areg'], callback = self.flight_base_parse, meta={'areg':aRegObj})
                yield scrapy.Request(self.radarbox_live_api + aRegObj['flightid'], callback=self.radarbox_live_parse,
                                     meta={'areg': aRegObj}, headers={'Origin': 'https://www.radarbox.com'})
            except:
                self.logger.error("调用radarbox_live_api异常")

    def radarbox_live_parse(self, response):
        item = FlightBaseItem()
        aRegObj = response.request.meta['areg']
        try:
            base_data = json.loads(response.text)
            if base_data is None:
                yield scrapy.Request(self.radarbox_history_api + aRegObj['areg'], callback=self.radarbox_history_parse,
                                     meta={'areg': aRegObj}, headers={'Origin': 'https://www.radarbox.com'})
                return
            fid = aRegObj['flightid']
            item['FlightID'] = '1#%s' % fid
            if 'cs' in base_data and base_data['cs'] is not None:
                item['ICAO'] = base_data['cs']
            if 'fnia' in base_data and base_data['fnia'] is not None:
                item['IATA'] = base_data['fnia']
            if 'acr' in base_data and base_data['acr'] is not None:
                item['PlaneReg'] = base_data['acr']
            if 'aporgci' in base_data and base_data['aporgci'] is not None:
                item['TakeoffBase'] = base_data['aporgci']
            if 'aporgic' in base_data and base_data['aporgic'] is not None:
                item['TakeoffBaseICAO'] = base_data['aporgic']
            if 'aporgia' in base_data and base_data['aporgia'] is not None:
                item['TakeoffBaseIATA'] = base_data['aporgia']
            if 'apdstci' in base_data and base_data['apdstci'] is not None:
                item['LandBase'] = base_data['apdstci']
            if 'apdstic' in base_data and base_data['apdstic'] is not None:
                item['LandBaseICAO'] = base_data['apdstic']
            if 'apdstia' in base_data and base_data['apdstia'] is not None:
                item['LandBaseIATA'] = base_data['apdstia']
            if 'svd' in base_data and base_data['svd'] is not None:
                item['LastTime'] = int(base_data['svd'])
            item['UpdateTime'] = str(int(time.time() * 1000))

            departure_relative = None
            duration = None
            if 'departureRelative' in base_data:
                departure_relative = base_data['departureRelative']
            if 'duration' in base_data:
                duration = base_data['duration']
            item['ScheduleBegTime'], item['ScheduleArrTime'] = self.get_schedule_time(
                item['LastTime'], departure_relative, duration)
            item['ActualBegTime'] = item['ScheduleBegTime']
            item['ActualArrTime'] = item['ScheduleArrTime']

            # 将已采集航班的飞机注册号存到已采集集合中，并设置过期时间
            expire = time.time() + 240 * 60
            self.redis_client.sadd('AirplaneSpiders:fidCrawled',
                                   json.dumps({'fid': item['FlightID'], 'expire': expire}))
            yield item
        except Exception as e:
            self.logger.error('航班基础信息解析错误，原因是：', e)

    def radarbox_history_parse(self, response):
        item = FlightBaseItem()
        try:
            base_data = None
            rsp = json.loads(response.text)
            # rsp = json.loads(response.xpath('//pre//text()').get())
            if rsp is None or 'flights' not in rsp and 'registrations' not in rsp:
                return
            aRegObj = response.request.meta['areg']
            fid = aRegObj['flightid']
            for flight in rsp['flights']:
                if fid == str(flight['fid']):
                    base_data = flight
                    break
            if base_data is None:
                for flight in rsp['registrations']:
                    if fid == str(flight['fid']):
                        base_data = flight
                        break
            if base_data is None:
                return
            item['FlightID'] = '1#%s' % fid
            if 'cs' in base_data and base_data['cs'] is not None:
                item['ICAO'] = base_data['cs']
            if 'fnia' in base_data and base_data['fnia'] is not None:
                item['IATA'] = base_data['fnia']
            if 'acr' in base_data and base_data['acr'] is not None:
                item['PlaneReg'] = base_data['acr']
            if 'aporgci' in base_data and base_data['aporgci'] is not None:
                item['TakeoffBase'] = base_data['aporgci']
            if 'aporgic' in base_data and base_data['aporgic'] is not None:
                item['TakeoffBaseICAO'] = base_data['aporgic']
            if 'aporgia' in base_data and base_data['aporgia'] is not None:
                item['TakeoffBaseIATA'] = base_data['aporgia']
            if 'apdstci' in base_data and base_data['apdstci'] is not None:
                item['LandBase'] = base_data['apdstci']
            if 'apdstic' in base_data and base_data['apdstic'] is not None:
                item['LandBaseICAO'] = base_data['apdstic']
            if 'apdstia' in base_data and base_data['apdstia'] is not None:
                item['LandBaseIATA'] = base_data['apdstia']
            # 起降时间及时区转换
            aporgtz = None
            apdsttz = None
            if 'aporgtz' in base_data and base_data['aporgtz'] is not None:
                aporgtz = base_data['aporgtz']
            if 'apdsttz' in base_data and base_data['apdsttz'] is not None:
                apdsttz = base_data['apdsttz']
            if 'depsts' in base_data and base_data['depsts'] is not None:
                item['ScheduleBegTime'] = int(base_data['depsts'] * 1000)
                if aporgtz is not None:
                    item['cccc'] = int(item['ScheduleBegTime'] - (aporgtz - 8) * 3600 * 1000)
            if 'depc' in base_data and base_data['depc'] is not None:
                item['ActualBegTime'] = int(base_data['depc'] * 1000)
                if aporgtz is not None:
                    item['ActualBegTime'] = int(item['ActualBegTime'] - (aporgtz - 8) * 3600 * 1000)
            if 'arrsts' in base_data and base_data['arrsts'] is not None:
                item['ScheduleArrTime'] = int(base_data['arrsts'] * 1000)
                if apdsttz is not None:
                    item['ScheduleArrTime'] = int(
                        item['ScheduleArrTime'] - (apdsttz - 8) * 3600 * 1000)
            if 'arrc' in base_data and base_data['arrc'] is not None:
                item['ActualArrTime'] = int(base_data['arrc'] * 1000)
                if apdsttz is not None:
                    item['ActualArrTime'] = int(item['ActualArrTime'] - (apdsttz - 8) * 3600 * 1000)

            if 'svd' in base_data and base_data['svd'] is not None:
                item['LastTime'] = int(base_data['svd'])
            item['UpdateTime'] = str(int(time.time() * 1000))

            # 将已采集航班的飞机注册号存到已采集集合中，并设置过期时间
            expire = time.time() + 240 * 60
            self.redis_client.sadd('AirplaneSpiders:fidCrawled',
                                   json.dumps({'fid': item['FlightID'], 'expire': expire}))
            yield item
        except Exception as e:
            self.logger.error('航班基础信息解析错误，原因是：', e)

    def varify_flight_parse(self, response):
        item = response.request.meta['item']
        base_data = json.loads(response.text)
        if base_data is not None and 'data' in base_data:
            base_data = base_data['data']
            # 时间转为北京时间#
            if 'orgTinezone' in base_data:
                tz = base_data['orgTinezone']
                if 'scheduledDeptime' in base_data and int(base_data['scheduledDeptime']) != 0:
                    item['ScheduleBegTime'] = (int(base_data['scheduledDeptime']) - tz + 3600 * 8) * 1000
                if 'actualDeptime' in base_data and int(base_data['actualDeptime']) != 0:
                    item['ActualBegTime'] = (int(base_data['actualDeptime']) - tz + 3600 * 8) * 1000
            if 'dstTinezone' in base_data:
                tz = int(base_data['dstTinezone'])
                if 'scheduledArrtime' in base_data and int(base_data['scheduledArrtime']) != 0:
                    item['ScheduleArrTime'] = (int(base_data['scheduledArrtime']) - tz + 3600 * 8) * 1000
                if 'actualArrtime' in base_data and int(base_data['actualArrtime']) != 0:
                    item['ActualArrTime'] = (int(base_data['actualArrtime']) - tz + 3600 * 8) * 1000

            # flightBaseItem['LastTime'] = str(int(time.time() * 1000))
            item['UpdateTime'] = str(int(time.time() * 1000))

        yield item

    def closed(self, reason):
        # self.redisclient = redis.Redis(host= self.settings['REDIS_HOST'], port= int(self.settings['REDIS_PORT']), password= self.settings['REDIS_PWD'])
        # for obj in self.crawledRegs:
        #     self.redisclient.sadd('aRegsCrawled', json.dumps(obj))
        self.redis_client.close()

    def get_schedule_time(self, current_time, departure_relative, duration):
        if not departure_relative:
            return 0, 0
        dr_result = re.findall(r'\d+\.?\d*', departure_relative)
        dr_seconds = 0
        if len(dr_result) == 2:
            dr_seconds = int(dr_result[0]) * 60 * 60 + int(dr_result[1]) * 60
        elif len(dr_result) == 1:
            dr_seconds = int(dr_result[0]) * 60
        departure_time = current_time - dr_seconds * 1000

        if not duration:
            return departure_time, 0
        duration_result = re.findall(r'\d+\.?\d*', duration)
        duration_seconds = 0
        if len(duration_result) == 2:
            duration_seconds = int(duration_result[0]) * 60 * 60 + int(duration_result[1]) * 60
        elif len(duration_result) == 1:
            duration_seconds = int(duration_result[0]) * 60
        arri_time = departure_time + duration_seconds * 1000

        return departure_time, arri_time
