# -*- coding: utf-8 -*-
import scrapy
import time
import json
import datetime
import re
from copy import deepcopy
from lxml import etree
from AirplaneSpiders.items import FlightBaseItem


# 航班基本信息采集--数据源：planemapper#
class FlightBase4Spider(scrapy.Spider):
    name = 'planemapper_flight_base'
    allowed_domains = ['planemapper.com']
    # start_urls = ['http://planemapper.com/']
    # post请求所有飞行中的飞机，提取flightId
    live_url = 'https://www.planemapper.com/php_ctr/flightmap_ctr.php'
    airport_url = 'https://www.planemapper.com/flights/'  # 使用航班号查询机场等信息，需要解析网页提取数据

    custom_settings = {
        'DOWNLOAD_DELAY': 0.2,
        'BATCH_SAVE_SIZE': 100,
        'PROTO_MODULE_PATH': 'AirplaneSpiders.proto.Plane_pb2',
        'PROTO_CLASS_NAME': 'FligthInfoSets',
        'PROTO_FIELD_NAME': 'FligthInfo',
        'PROTO_SAVE_FILE_PATH': r'/usr/local/spider_data',
        'PROTO_SAVE_FILE_NAME': r'track_data_flight_',
        'ITEM_PIPELINES': {
            'AirplaneSpiders.pipelines.ProtobufSavePipeline': 300
        }
    }

    def __init__(self, params=None, *args, **kwargs):
        super(FlightBase4Spider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def start_requests(self):
        yield scrapy.FormRequest(url=self.live_url, formdata={'command': '5'}, callback=self.parse)

    def parse(self, response):
        try:
            live_data = json.loads(response.text)
            for data in live_data:
                item = FlightBaseItem()
                item['FlightID'] = '4#%s' % data[0]
                item['PlaneReg'] = data[2]
                item['UpdateTime'] = str(int(time.time() * 1000))
                item['LastTime'] = str(int(time.time() * 1000))

                form_data = {'command': '2', 'Icao': data[0]}
                yield scrapy.FormRequest(url=self.live_url, formdata=form_data, callback=self.parse_flight_no,
                                         meta={'item': deepcopy(item)})
                # yield scrapy.Request(url=self.plane_live_url, callback=self.parsePlaneLiveInfo, method='POST', body=json.dumps(formdata), meta={'item', item})
        except Exception as e:
            self.logger.error("来自数据源planemapper的飞机实时数据解析失败，原因是：", e)

    def parse_flight_no(self, response):
        item = response.request.meta['item']
        try:
            base_data = json.loads(response.text)[0]
            flight_no = base_data.split('|')[0]  # 提取航班号用于查询航班信息
            if flight_no != '':
                yield scrapy.Request(url=self.airport_url + flight_no, meta={'item': deepcopy(item)},
                                     callback=self.parse_airport_info)
            else:
                yield item
        except Exception as e:
            self.logger.error("来自数据源planemapper的航班号解析失败，原因是：", e)

    def parse_airport_info(self, response):
        item = response.request.meta['item']
        try:
            selector = etree.HTML(response.text)

            item['TakeoffBase'] = selector.xpath("//div[@itemprop='departureAirport']/meta[@itemprop='name']/@content")[
                0]
            item['TakeoffBaseIATA'] = \
            selector.xpath("//div[@itemprop='departureAirport']/meta[@itemprop='iataCode']/@content")[0]
            item['TakeoffBaseICAO'] = \
            selector.xpath("//div[@itemprop='departureAirport']/meta[@itemprop='icaoCode']/@content")[0]
            item['LandBase'] = selector.xpath("//div[@itemprop='arrivalAirport']/meta[@itemprop='name']/@content")[0]
            item['LandBaseIATA'] = \
            selector.xpath("//div[@itemprop='arrivalAirport']/meta[@itemprop='iataCode']/@content")[0]
            item['LandBaseICAO'] = \
            selector.xpath("//div[@itemprop='arrivalAirport']/meta[@itemprop='icaoCode']/@content")[0]
            departure_time = selector.xpath("//meta[@itemprop='departureTime']/@content")[0]
            item['ScheduleBegTime'] = self.time_util(departure_time)
            arrival_time = selector.xpath("//meta[@itemprop='arrivalTime']/@content")[0]
            item['ScheduleArrTime'] = self.time_util(arrival_time)

            yield item
        except Exception as selector:
            self.logger.error("来自数据源planemapper的机场信息解析失败， 原因是：", selector)

    # 将时间转为utc8并转为时间戳
    def time_util(self, time_str):
        tag = 8 - int(re.findall(r'.+([+,-]{1}.+?):', time_str)[0])  # 与utc8的时差
        time_str = re.findall(r'(.+T.+)[-,+]{1}.+', time_str)[
            0]  # 去掉2020-09-19T00:20:00+2:00后面的+2:00 => 2020-09-19T00:20:00
        utc8_time = datetime.datetime.strptime(time_str, r'%Y-%m-%dT%H:%M:%S') + datetime.timedelta(hours=tag)  # utc8时间
        str1 = time.strptime(str(utc8_time), r'%Y-%m-%d %H:%M:%S')
        timestamp = str(int(time.mktime(str1) * 1000))
        return timestamp
