# -*- coding: utf-8 -*-
import scrapy
import time
import json
from lxml import etree
from copy import deepcopy
from AirplaneSpiders.items import AirPlaneBaseItem


# 全球飞机基本信息采集引擎--数据源planemapper.com#
class AirplaneBase4Spider(scrapy.Spider):
    name = 'planemapper_airplane_base'
    allowed_domains = ['planemapper.com']
    # start_urls = ['http://planemapper.com/']

    live_url = 'https://www.planemapper.com/php_ctr/flightmap_ctr.php'  # post请求所有飞行中的飞机
    plane_base_url = 'https://www.planemapper.com/aircrafts/'  # 使用注册号查询飞机的基本信息，需要解析网页提取数据

    custom_settings = {
        'PROTO_MODULE_PATH': 'AirplaneSpiders.proto.Plane_pb2',
        'PROTO_CLASS_NAME': 'PlaneInfoSets',
        'PROTO_FIELD_NAME': 'PlaneInfo',
        'PROTO_SAVE_FILE_PATH': r'/usr/local/spider_data',
        'PROTO_SAVE_FILE_NAME': r'target_data_plane_',
        'IMAGES_STORE': r'/usr/local/temp_image/plane',
        'IMAGES_RESULT_FIELD': 'Image',
        'ZIP_FILE_NAME': 'image_data_plane_',
        'BATCH_SAVE_SIZE': 100,
        'ITEM_PIPELINES': {
            'scrapy.pipelines.images.ImagesPipeline': 1,
            'AirplaneSpiders.pipelines.ProtobufSavePipeline': 300
        }
    }

    def __init__(self, params=None, *args, **kwargs):
        super(AirplaneBase4Spider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def start_requests(self):
        yield scrapy.FormRequest(url=self.live_url, formdata={'command': '5'}, callback=self.parse)

    def parse(self, response):
        try:
            live_data = json.loads(response.text)
            for data in live_data:
                item = AirPlaneBaseItem()
                item['Reg'] = data[2]
                item['DataSource'] = 'planemapper'
                item['UpdateTime'] = str(int(time.time() * 1000))
                item['LastTime'] = str(int(time.time() * 1000))
                if data[1] is not None:
                    item['Callsign'] = data[1]

                yield scrapy.Request(url=self.plane_base_url + item['Reg'], callback=self.parse_plane_info,
                                     meta={'item': deepcopy(item)})
        except Exception as e:
            self.logger.error("来自数据源planemapper的飞机实时数据解析失败，原因是：", e)

    def parse_plane_info(self, response):
        item = response.meta['item']
        try:
            selector = etree.HTML(response.text)
            item['Owner'] = selector.xpath("//*[@id='body']/div[1]/div/div[1]/div[7]/div[1]/p[2]/a")[0].text
            # item['Nation'] = e.xpath("").extract_first()
            item['Model'] = selector.xpath("//meta[contains(@itemprop, 'model')]/@content")[0]
            if 'url' in selector.xpath("//meta[contains(@property, 'image')]/@content"):
                item['image_urls'] = []
                url = selector.xpath("//meta[contains(@property, 'image')]/@content").url
                item['image_urls'].append(url)
            item['Age'] = selector.xpath("//*[@id='body']/div[1]/div/div[1]/div[7]/div[5]/p[2]")[0].text
            yield item
        except Exception as selector:
            self.logger.error("来自数据源planemapper的飞机基础信息解析失败，原因是：", selector)
