import scrapy
from scrapy import Request
import re
import time
from AirplaneSpiders.items import NotamInfo
import json
from AirplaneSpiders.utils import locationTranslate, dateTranslate


# 航空管制区采集引擎
class AirTrafficSpider(scrapy.Spider):
    name = 'tfr_air_traffic'
    allowed_domains = ['faa.gov']
    custom_settings = {
        'PROTO_MODULE_PATH': 'AirplaneSpiders.proto.Notam_pb2',
        'PROTO_CLASS_NAME': 'NotamInfoSets',
        'PROTO_FIELD_NAME': 'NotamInfo',
        'PROTO_SAVE_FILE_PATH': r'/usr/local/spider_data',
        'PROTO_SAVE_FILE_NAME': r'target_data_notam_',
        'IMAGES_STORE': r'/usr/local/temp_image/trafic',
        'IMAGES_URLS_FIELD': 'Image',
        'IMAGES_RESULT_FIELD': 'Image',
        'ZIP_FILE_NAME': 'image_data_notam_',
        'ITEM_PIPELINES': {
            'scrapy.pipelines.images.ImagesPipeline': 1,
            'AirplaneSpiders.pipelines.ProtobufSavePipeline': 300
        }
    }

    def __init__(self, params=None, *args, **kwargs):
        super(AirTrafficSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def start_requests(self):
        yield Request('https://tfr.faa.gov/tfr2/list.html')

    def parse(self, response):
        date_list = response.xpath('//tr/td[1]/a[contains(@href,"save_pages")]')
        for date in date_list:
            date_url = date.xpath('./@href').get()
            pubDate = date.xpath('./text()').get()
            data_url = 'https://tfr.faa.gov%s' % date_url[2:]
            yield Request(data_url, callback=self.parse_detail, meta={'pubDate': pubDate})
        # yield Request('https://tfr.faa.gov/save_pages/detail_0_6471.html',callback=self.parse_detail)

    def parse_detail(self, response):
        notam_item = NotamInfo()
        # pubDate = response.request.meta['pubDate']
        # pubDate = int(time.mktime(time.strptime(pubDate,'%m/%d/%Y')) * 1000)
        # notamItem['Lasttime'] = pubDate

        notam_number = response.xpath(
            '//font[text()="NOTAM Number     :"]/../../following-sibling::td/b/font/text()').get()
        notam_item['NotamNumber'] = notam_number

        issue_date = response.xpath('//td//font[contains(text(),"Issue")]/parent::*')
        issue_date = issue_date.xpath('./parent::*').xpath('./td[2]/font/text()').get()
        if issue_date is not None:
            notam_item['IssueDate'] = int(dateTranslate(issue_date) * 1000)

        location = response.xpath('//td//font[contains(text(),"Location")]/parent::*')
        location = location.xpath('./parent::*').xpath('./td[2]/font/text()').get()
        notam_item['Location'] = location

        beginning = response.xpath('//td//font[contains(text(),"Beginning")]/parent::*')
        beginning = beginning.xpath('./parent::*').xpath('./td[2]/font/text()').get()
        if beginning is not None:
            if beginning.find('Immediately') >= 0:
                notam_item['BeginningDatetime'] = notam_item['IssueDate']
            else:
                notam_item['BeginningDatetime'] = int(dateTranslate(beginning) * 1000)

        ending = response.xpath('//td//font[contains(text(),"Ending")]/parent::*')
        ending = ending.xpath('./parent::*').xpath('./td[2]/font/text()').get()
        if ending is not None:
            if ending.find('Permanent') >= 0:
                notam_item['EndingDateTime'] = -1
            else:
                notam_item['EndingDateTime'] = int(dateTranslate(ending) * 1000)

        type_str = response.xpath('//td//font[contains(text(),"Type")]/parent::*')
        type_str = type_str.xpath('./parent::*').xpath('./td[2]/font/text()').get()
        notam_item['Type'] = type_str

        reason = response.xpath('//td//font[contains(text(),"Reason")]/parent::*')
        reason = reason.xpath('./parent::*').xpath('./td[2]/font/text()').get()
        notam_item['Reason'] = reason

        # Replaced = response.xpath('//td//font[contains(text(),"Replaced")]/parent::*')
        # Replaced = Replaced.xpath('./parent::*').xpath('./td[2]/font/text()').get()
        # notamItem[''] =

        # Pilots = response.xpath('//td//font[contains(text(),"Pilots")]/parent::*')
        # Pilots = Pilots.xpath('./parent::*').xpath('./td[2]/font/text()').get()
        # notamItem[''] =

        images = response.xpath('//img/@src').getall()
        imageUrls = []
        for image in images:
            if image.find('save_maps') >= 0:
                imageUrls.append('https://tfr.faa.gov%s' % image)
        notam_item['Image'] = imageUrls
        notam_item['EffectiveDates'] = []
        # 航空管制区类型识别及区域点位
        affectAreas = self.affect_area_info(response, notam_item)

        notam_item['EffectiveDates'] = json.dumps(notam_item['EffectiveDates'])
        notam_item['AffectedAreas'] = json.dumps(affectAreas)
        notam_item['UpdateTime'] = str(int(time.time() * 1000))

        print(notam_item)

        # yield notam_item

    # 航空管制区类型识别及区域点位
    def affect_area_info(self, response, notam_item):
        affectAreas = []
        common_path = '//td//font[contains(text(),"%s")]/../..'
        area_type = ["From:", "Center:", "Line Segments:"]
        from_to_type = response.xpath(common_path % area_type[0])
        center_type = response.xpath(common_path % area_type[1])
        line_type = response.xpath(common_path % area_type[2])
        if from_to_type is not None and len(from_to_type) > 0:
            affectAreas.append(self.airspace_from_to(from_to_type, notam_item))
        if center_type is not None and len(center_type) > 0:
            affectAreas.append(self.airspace_center(center_type, notam_item))
        if line_type is not None and len(line_type) > 0:
            affectAreas.append(self.airspace_line(line_type, notam_item))
        return affectAreas

    def airspace_from_to(self, area_type_selector, notam_item):
        if area_type_selector is None or len(area_type_selector) == 0:
            return {}
        notam_item['AffectedAreaType'] = 1
        for from_tr in area_type_selector:
            points = []
            from_latitude = locationTranslate(from_tr.xpath('./td[4]/font/text()').get())
            from_longitude = locationTranslate(from_tr.xpath('./td[5]/font/text()').get())
            # fromFrd = fromTr.xpath('./td[6]/font/text()').get()
            points.append({'latitude': from_latitude, 'longitude': from_longitude})
            airspace_to_list = from_tr.xpath('./following-sibling::*')
            for airspace_to in airspace_to_list:
                to_latitude = locationTranslate(airspace_to.xpath('./td[4]/font/text()').get())
                to_longitude = locationTranslate(airspace_to.xpath('./td[5]/font/text()').get())
                # toFrd = toTr.xpath('./td[6]/font/text()').get()
                points.append({'latitude': to_latitude, 'longitude': to_longitude})
            # 获取海拔信息
            ancestor_tables = from_tr.xpath('./ancestor::table')
            ancestor_tr = ancestor_tables[len(ancestor_tables) - 1].xpath('./ancestor::tr')
            sibling_tr = ancestor_tr[len(ancestor_tr) - 1].xpath('./following-sibling::*')
            td3 = sibling_tr[0].xpath('./td[3]')
            altitude_info = td3.xpath('./font/text()').get()
            altitudes = []
            altitude_number = re.findall(r'\d+', altitude_info)
            for number in altitude_number:
                altitudes.append(number)
            effective_date = ''
            index = 2
            while index < len(sibling_tr):
                effective_info = sibling_tr[index].xpath('./td[3]/font/text()').get()
                if effective_info is not None:
                    effective_date += effective_info + '\n'
                else:
                    break
                index += 1
            notam_item['EffectiveDates'].append(effective_date)
            return {'points': points, 'altitudes': altitudes}

    def airspace_center(self, area_type_selector, notam_item):
        if area_type_selector is None or len(area_type_selector) == 0:
            return {}
        notam_item['AffectedAreaType'] = 0
        for centerTr in area_type_selector:
            center_info = centerTr.xpath('./td[4]/font/text()').get()
            latitude_match = re.search('Latitude: (.*),', center_info)
            latitude = locationTranslate(latitude_match.group(1))
            longitude_match = re.search(r'Longitude: (.*)\)', center_info)
            longitude = locationTranslate(longitude_match.group(1))
            radiusTr = centerTr.xpath('./following-sibling::*')[0]
            altitudeTr = centerTr.xpath('./following-sibling::*')[1]
            radiusText = radiusTr.xpath('./td[4]/font/text()').get()
            altitudeText = altitudeTr.xpath('./td[4]/font/text()').get()
            radius = re.search(r'(\d+)', radiusText).group(1)
            altitudes = []
            altitude_number = re.findall(r'\d+', altitudeText)
            for number in altitude_number:
                altitudes.append(number)
            index = 3
            effective_date = ''
            while index < len(centerTr.xpath('./following-sibling::*')):
                effectiveTr = centerTr.xpath('./following-sibling::*')[index]
                effective_info = effectiveTr.xpath('./td[3]/font/text()').get()
                if effective_info is not None:
                    effective_date += '\n' + effective_info
                index += 1
            notam_item['EffectiveDates'].append(effective_date)
            return {'latitude': latitude, 'longitude': longitude, 'radius': radius, 'altitudes': altitudes}

    def airspace_line(self, area_type_selector, notam_item):
        if area_type_selector is None or len(area_type_selector) == 0:
            return {}
        notam_item['AffectedAreaType'] = 2
        for lineSettingTr in area_type_selector:
            pointsTrs = lineSettingTr.xpath('./following-sibling::*')[0].xpath('./td[2]//table//tr')
            index = 1
            points = []
            while index < len(pointsTrs):
                pointTr = pointsTrs[index]
                latitude = locationTranslate(pointTr.xpath('./td[1]/font/text()').get())
                longitude = locationTranslate(pointTr.xpath('./td[2]/font/text()').get())
                index += 1
                points.append({'longitude': longitude, 'latitude': latitude})
            bufferTr = lineSettingTr.xpath('./preceding-sibling::*')[-2]
            bufferText = bufferTr.xpath('./td[3]/font/text()').get()
            buffer = re.search(r'(\d+\.?\d*)', bufferText).group(1)
            altitudeTr = lineSettingTr.xpath('./following-sibling::*')[1]
            altitudes = []
            altitude_number = re.findall(r'\d+', altitudeTr.xpath('./td[2]/font/text()').get())
            for number in altitude_number:
                altitudes.append(number)
            index = 3
            effective_date = ''
            while index < len(lineSettingTr.xpath('./following-sibling::*')):
                effectiveTr = lineSettingTr.xpath('./following-sibling::*')[index]
                effective_info = effectiveTr.xpath('./td[3]/font/text()').get()
                if effective_info is not None:
                    effective_date += '\n' + effective_info
                index += 1
            notam_item['EffectiveDates'].append(effective_date)
            return {'points': points, 'buffer': buffer, 'altitudes': altitudes}
