import json
import logging as logger
import random
import time
import uuid

import redis
import requests
import scrapy
from scrapy.utils.project import get_project_settings
from selenium.common.exceptions import WebDriverException
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.firefox.options import Options

from ShipSpiders.items import ShipInfoItem
from ShipSpiders.scrapy_selenium import SeleniumRequest
from ShipSpiders.utils.country_digit import country_digit_dict


class ShipSpider(scrapy.Spider):
    name = 'shipxy_info'
    settings = get_project_settings()
    custom_settings = {
        'PROTO_MODULE_PATH': 'ShipSpiders.proto.Ship_pb2',
        'PROTO_CLASS_NAME': 'ShipInfoSets',
        'PROTO_FIELD_NAME': 'ShipInfo',
        'PROTO_SAVE_FILE_NAME': 'target_data_ship_',
        'PROTO_SAVE_FILE_PATH': r'/usr/local/spider_data',
        'FILE_TRANS_PATH': r'/usr/local/spider_data',
        'IMAGES_STORE': r'/usr/local/temp_image/ship',
        'IMAGES_RESULT_FIELD': 'Image',
        'ZIP_FILE_NAME': 'image_data_ship_',
        'MAP_RESOLUSION': 1,
        'ITEM_PIPELINES': {
            'ShipSpiders.pipelines.ProtobufSavePipeline': 300,
            'scrapy.pipelines.images.ImagesPipeline': 2,
        },
        'EXTENSIONS': {},
    }
    image_urls = []
    photo_query_url = 'http://shippic.shipxy.com/getshippiclist.ashx?mmsi={mmsi}&imo={imo}&shipname=0'
    redisClient = redis.Redis(host=settings['REDIS_HOST'],
                              port=int(settings['REDIS_PORT']),
                              password=settings['REDIS_PWD'])

    def __init__(self, params=None, *args, **kwargs):
        super(ShipSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def start_requests(self):
        self.driver_options = Options()
        self.driver_options.add_argument('-headless')
        self.total_ship_count = 0
        self.info_request_times = 0
        self.get_ship_request_times = 0
        self.cached_info_read_times = 0
        self.total_progress = '0%'
        self.sensitive_mmsi = self.redisClient.smembers('ShipSpiders:sensitive_mmsi')
        yield SeleniumRequest(url='http://www.shipxy.com/',
                              callback=self.next_requests)

    def next_requests(self, response):
        self.driver = response.request.meta['driver']
        try:
            self.driver.get('http://www.shipxy.com/Home/Login')
            time.sleep(2)
            logger.info('Logging in with user_id and password...')
            pwdbutton = self.driver.find_element_by_xpath("//a[text()='密码登录']")
            pwdbutton.click()
            time.sleep(0.5)
            userName = self.driver.find_element_by_id('userName')
            userPWD = self.driver.find_element_by_id('userPWD')
            button = self.driver.find_element_by_id('loginBtn')
            userName.send_keys('jlit_yuxin')
            userPWD.send_keys('passok')
            button.click()
            time.sleep(3)
            self.driver.get('http://www.shipxy.com/')
            logger.info('Logged in! Updating cookies...')
        except:
            pass
        self.dictCookie = self.driver.get_cookies()

        logger.info('Getting %d sensitive ship info' % len(self.sensitive_mmsi))
        for mmsi in self.sensitive_mmsi:
            params = {'shipIDs': mmsi, 'MMSI': mmsi}
            yield scrapy.FormRequest(url='http://www.shipxy.com/ship/GetShip',
                                     callback=self.parse_ship,
                                     formdata=params,
                                     cookies=self.dictCookie)

        logger.info('Getting normal ship info...')
        slat, elat = -90 * 1000000, 90 * 1000000
        slon, elon = -180 * 1000000, 180 * 1000000
        self.global_area = (elon - slon) * (elat - slat)
        self.total_area = 0
        url = 'http://www.shipxy.com/ship/getareashipssimple?level=11&area={slon},{slat},{elon},{elat}&enc=0&src=0&_t={time}'
        yield scrapy.Request(url=url.format(slon=slon,
                                            slat=slat,
                                            elon=elon,
                                            elat=elat,
                                            time=int(time.time() * 1000)),
                             callback=self.parse,
                             cookies=self.dictCookie,
                             meta={
                                 'slon': slon,
                                 'slat': slat,
                                 'elon': elon,
                                 'elat': elat
                             })

    def parse(self, response, **kwargs):
        rsp_obj = json.loads(response.text)
        slon = response.meta['slon']
        slat = response.meta['slat']
        elon = response.meta['elon']
        elat = response.meta['elat']
        current_area = 'Latitude: %.3f ~ %.3f, Longitude: %.3f ~ %.3f' % (
            slat / 1000000, elat / 1000000, slon / 1000000, elon / 1000000)
        if rsp_obj['status'] != 0:
            logger.warning('Response no data! %s' % response.text)
            return
        _count = rsp_obj['count']
        resolution = self.settings['MAP_RESOLUSION']
        if _count < 1524 or elon - slon < resolution * 1000000:  # 设置最小区域范围
            self.total_area += (elat - slat) * (elon - slon)
            self.total_progress = '%.2f%%' % float(self.total_area / self.global_area * 100)
            if rsp_obj['status'] != 0:
                logger.warning('Response no data! %s' % response.text)
                return
            _count = rsp_obj['count']
            if rsp_obj['count'] == 0:
                logger.info('[Area Covered: %s] 0 ship found in area %s' % (self.total_progress, current_area))
                return
            data = rsp_obj['data']
            self.total_ship_count += _count
            logger.info('[Area Covered: %s] %d ship(s) found in area %s' % (self.total_progress, _count, current_area))
            decode_api = self.settings['DECODE_SERVICE']
            retry_times = 0
            while retry_times <= 3:
                try:
                    decode_data = requests.post(decode_api, data=data)
                    if decode_data.content == b'500':
                        retry_times += 1
                        logger.warning("shipxy_decode server error! Restarting server...")
                        continue
                    else:
                        ships = json.loads(decode_data.content)
                        break
                except:
                    retry_times += 1
                    logger.warning("Decode failure! Retrying for %d time(s) after 3 seconds..."
                                   % retry_times)
                    time.sleep(3)

            if retry_times > 3:
                logger.warning("%d ship data loss for area %s" % (_count, current_area))
                return

            for _ship in ships:
                update_info = True
                if _ship['mmsi'] is None:
                    continue
                if self.redisClient.hexists(name='ship_info', key=_ship['mmsi']):
                    info_bstr = self.redisClient.hget(name='ship_info',
                                                      key=_ship['mmsi'])
                    info = json.loads(info_bstr.decode())
                    current_timestamp_milli = int(round(time.time() * 1000))
                    if current_timestamp_milli > info['ExpireTimestamp']:
                        update_info = True
                    else:
                        update_info = False
                else:
                    update_info = True

                if update_info:
                    update_cookies_times = self.settings['UPDATE_COOKIES_AFTER_REQUEST_COUNT']
                    if self.info_request_times > update_cookies_times:
                        logger.info('Get %d ship info from shipxy' % update_cookies_times)
                        session_start_times = 1
                        while session_start_times <= 3:
                            try:
                                self.driver.quit()
                            except:
                                logger.info('Old session quit error! Start new session anyway...')

                            try:
                                self.driver.start_session(capabilities=DesiredCapabilities.FIREFOX)
                                logger.info('New session started! Refreshing shipxy.com webpage...')
                                self.driver.get('http://www.shipxy.com/')
                                break
                            except WebDriverException:
                                logger.info(
                                    'Session start error! retrying start new session for %d time(s) atfer 3 seconds...' % session_start_times)
                                session_start_times += 1
                                time.sleep(3)

                        if session_start_times > 3:
                            logger.info('Cannot start new session! Please check webdriver')
                        else:
                            logger.info('Updating cookies...')
                            time.sleep(1)
                            self.dictCookie = self.driver.get_cookies()
                            self.info_request_times = 0
                    mmsi = str(_ship['mmsi'])
                    params = {'shipIDs': mmsi, 'MMSI': mmsi}
                    yield scrapy.FormRequest(
                        url='http://www.shipxy.com/ship/GetShip',
                        callback=self.parse_ship,
                        formdata=params,
                        cookies=self.dictCookie)
                else:
                    self.cached_info_read_times += 1
                    if self.cached_info_read_times >= 3000:
                        logger.info('Loaded 3000 Ship cached info from REDIS')
                        self.cached_info_read_times = 0
                    item = ShipInfoItem()
                    item['ID'] = str(uuid.uuid4())
                    for _ in info:
                        item[_] = info[_]
                    item['UpdateTime'] = current_timestamp_milli
                    yield item
        else:
            # logger.info('Ship count exceed in area %s, search for subareas' % current_area)
            mlat = int((slat + elat) / 2)
            mlon = int((slon + elon) / 2)
            area = []
            area.append({'slon': slon, 'slat': slat, 'elon': mlon, 'elat': mlat})
            area.append({'slon': mlon, 'slat': slat, 'elon': elon, 'elat': mlat})
            area.append({'slon': slon, 'slat': mlat, 'elon': mlon, 'elat': elat})
            area.append({'slon': mlon, 'slat': mlat, 'elon': elon, 'elat': elat})
            url = 'http://www.shipxy.com/ship/getareashipssimple?level=11&area={slon},{slat},{elon},{elat}&enc=0&src=0&_t={time}'
            for sub_area in area:
                yield scrapy.Request(url=url.format(slat=sub_area['slat'],
                                                    slon=sub_area['slon'],
                                                    elat=sub_area['elat'],
                                                    elon=sub_area['elon'],
                                                    time=int(time.time() * 1000)),
                                     callback=self.parse,
                                     cookies=self.dictCookie,
                                     meta={'slat': sub_area['slat'], 'slon': sub_area['slon'], 'elat': sub_area['elat'],
                                           'elon': sub_area['elon']})

    def parse_ship(self, response):
        info = {}
        self.info_request_times += 1
        current_timestamp_milli = int(round(time.time() * 1000))
        try:
            ship_obj = json.loads(response.text)
            if ship_obj['status'] == 0:
                ships = ship_obj['data']
                for _ship in ships:
                    item = ShipInfoItem()
                    if 'shipid' in _ship:
                        info['ID'] = _ship['shipid']
                    if 'name' in _ship:
                        info['Name'] = _ship['name']
                    if 'imo' in _ship:
                        info['IMO'] = _ship['imo']
                    if 'mmsi' in _ship:
                        info['MMSI'] = _ship['mmsi']
                        _country = self.get_reg_country(str(_ship['mmsi'])[0:3], country_digit_dict)
                        if _country:
                            info['RegCountry'] = _country
                        if 'imo' in _ship:
                            image = self.get_image_url(_ship['mmsi'], _ship['imo'])
                            if image['status'] == 0:
                                info['image_urls'] = []
                                info['image_urls'].append(image['data'])
                    if 'callsign' in _ship:
                        info['Callsign'] = _ship['callsign']
                    # Owner
                    # Builder
                    # BuilderTime
                    if 'type' in _ship:
                        info['SourceType'] = _ship['type']
                    # UsageType （军民属性）
                    # Nettonnage
                    if 'width' in _ship:
                        info['Width'] = _ship['width'] / 10
                    if 'length' in _ship:
                        info['Length'] = _ship['length'] / 10
                    if 'left' in _ship:
                        info['LeftPost'] = _ship['left'] / 10
                    if 'trail' in _ship:
                        info['Trail'] = _ship['trail'] / 10
                    if 'draught' in _ship:
                        info['Draught'] = _ship['draught'] / 1000
                    # CrewNum
                    if 'lastdyn' in _ship:
                        info['LastTime'] = _ship['lastdyn'] * 1000
                    info['DataSource'] = 'shipxy'
                    info['UpdateTime'] = int(round(time.time() * 1000))

                    for _ in info:
                        item[_] = info[_]
                    info['ExpireTimestamp'] = current_timestamp_milli + (
                            24 * 60 + random.randint(0, 24 * 60)) * 60 * 1000  # 24-48小时过期
                    self.redisClient.hset('ship_info', info['MMSI'], json.dumps(info, ensure_ascii=False))
                    yield item
        except:
            logger.error(ship_obj)

    def get_reg_country(self, mmsi, country_dict):
        if mmsi in country_dict:
            return country_dict[mmsi]
        return None

    def get_image_url(self, mmsi, imo):
        photo_url = self.photo_query_url.format(mmsi=mmsi, imo=imo)
        response = requests.get(photo_url)
        data = json.loads(response.text)
        if 'status' in data and data['status'] == 1:
            return {
                'status': 1,
            }
        else:
            first_pic = data[0]
            # self.image_urls.append(first_pic['url'])
            return {
                'status': 0,
                'data': first_pic['surl']  # 全尺寸图片字段为url，缩略图的字段为surl
            }
