import json
import logging as logger
import random
import time
import uuid

import redis
import requests
import scrapy
from global_land_mask import globe
from scrapy.utils.project import get_project_settings
from selenium.common.exceptions import WebDriverException
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.firefox.options import Options

from ShipSpiders.items import VoyageTrackpointInfoItem
from ShipSpiders.scrapy_selenium import SeleniumRequest
from ShipSpiders.utils.http_utils import http_post
from ShipSpiders.utils.time_utils import to_unix_timestamp


class TrackpointsSpider(scrapy.Spider):
    name = 'shipxy_track'
    settings = get_project_settings()
    custom_settings = {
        'PROTO_MODULE_PATH': 'ShipSpiders.proto.Ship_pb2',
        'PROTO_CLASS_NAME': 'VoyageTrackpointInfoSets',
        'PROTO_FIELD_NAME': 'VoyageTrackpointInfo',
        'PROTO_SAVE_FILE_PATH': r'/usr/local/spider_data/',
        'PROTO_SAVE_FILE_NAME': r'track_data_ship_',
        'MAP_RESOLUTION': 15,
        'ITEM_PIPELINES': {
            'ShipSpiders.pipelines.ProtobufSavePipeline': 300
        }
    }
    redis_client = redis.Redis(host=settings['REDIS_HOST'],
                               port=int(settings['REDIS_PORT']),
                               password=settings['REDIS_PWD'])

    def __init__(self, params=None, *args, **kwargs):
        super(TrackpointsSpider, self).__init__(*args, **kwargs)
        self.driver_options = Options()
        self.driver_options.add_argument('-headless')
        self.total_ship_count = 0
        self.land_points = 0
        self.voyage_request_times = 0
        self.cached_voyage_read_times = 0
        self.updated_btime = 0
        self.total_progress = '0%'
        self.shipxy_voyage = self.redis_client.hgetall('shipxy_voyage')
        self.new_voyage = {}
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def start_requests(self):
        logger.info("Updating REDIS voyage cache, current cache length is %d" % len(self.shipxy_voyage))
        current_timestamp_milli = int(round(time.time() * 1000))
        if self.shipxy_voyage is not None and len(self.shipxy_voyage) > 0:
            for mmsi in list(self.shipxy_voyage.keys()):
                voyage_bstr = self.shipxy_voyage[mmsi]
                voyage = json.loads(voyage_bstr.decode())
                if current_timestamp_milli > voyage['ExpireTimestamp']:
                    self.shipxy_voyage.pop(mmsi)
        logger.info("Update finished, current cache length is %d" % len(self.shipxy_voyage))
        yield SeleniumRequest(url='https://www.baidu.com/', callback=self.next_requests)

    def next_requests(self, response):
        self.driver = response.request.meta['driver']
        try:
            self.driver.get('https://www.shipxy.com/Home/Login')
            time.sleep(2)
            logger.info('Logging in with user_id and password...')
            pwdbutton = self.driver.find_element_by_xpath("//a[text()='密码登录']")
            pwdbutton.click()
            time.sleep(0.5)
            userName = self.driver.find_element_by_id('userName')
            userPWD = self.driver.find_element_by_id('userPWD')
            button = self.driver.find_element_by_id('loginBtn')
            userName.send_keys(self.settings['SHIPXY_LOGIN_ACCOUNT'])
            userPWD.send_keys(self.settings['SHIPXY_LOGIN_PASSWD'])
            button.click()
            time.sleep(5)
            self.driver.get('https://www.shipxy.com/IHS')
            logger.info('Logged in! Updating cookies...')
        except:
            pass
        self.dict_cookie = self.driver.get_cookies()

        # logger.info('Getting normal ship trackpoints...')
        # slat, elat = -90 * 1000000, 90 * 1000000
        # slon, elon = -180 * 1000000, 180 * 1000000
        # self.global_area = (elon - slon) * (elat - slat)
        # self.total_area = 0
        # url = 'https://www.shipxy.com/ship/getareasimple?scode=0&level=11&area={slon},{slat},{elon},{elat}&enc=0&src=0&datatype=1&_t={time}'
        # yield scrapy.Request(url=url.format(slon=slon,
        #                                     slat=slat,
        #                                     elon=elon,
        #                                     elat=elat,
        #                                     time=int(time.time() * 1000)),
        #                      callback=self.parse,
        #                      cookies=self.dict_cookie,
        #                      meta={
        #                          'slon': slon,
        #                          'slat': slat,
        #                          'elon': elon,
        #                          'elat': elat
        #                      })

        sensitive_ship_query_api = self.settings['SENSITIVE_SHIP_QUERY_API']
        query_data_body = {
            "targetType": "ship",
            "delFlag": "0"
        }
        sensitive_target_info = http_post(sensitive_ship_query_api, json.dumps(query_data_body))
        rsp_body = json.loads(sensitive_target_info.text)
        if rsp_body['code'] == 200:
            sensitive_target_list = rsp_body['content']
            sensitive_target_count = int(rsp_body['message'])
            logger.info('获取 %d 艘高敏船舶的历史轨迹...' % sensitive_target_count)
            etime = int(time.time())  # 该API需要的时间戳是10位
            btime = int(int(self.redis_client.get("ShipSpiders:sensitive_latest_crawl_ts").decode()) / 1000)
            if btime < int(time.time() - 3600 * 24 * 30):
                btime = int(time.time() - 3600 * 24)
            logger.info("高敏目标轨迹采集开始时间为 {}".format(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(btime))))
            self.updated_btime = btime * 1000
            for sensitive_target in sensitive_target_list:
                mmsi = sensitive_target['targetValue']
                track_api = f'https://www.shipxy.com/Ship/GetTrackAll' \
                            f'?shipid={mmsi}&btime={btime}&etime={etime}&limit=1&enc=0'
                yield scrapy.Request(url=track_api, callback=self.parse_sensitive_ship, cookies=self.dict_cookie,
                                     meta={'mmsi': mmsi})

    def parse_sensitive_ship(self, response):
        mmsi = response.meta['mmsi']
        rsp_obj = json.loads(response.text)
        if rsp_obj['status'] != 0:
            # logger.info('[SENSITIVE SHIP] No track data of sensitive ship MMSI: %s' % mmsi)
            return
        data = rsp_obj['data']
        tracks = []
        track_decode_api = self.settings['TRACK_DECODE_SERVICE']
        retry_times = 1
        while retry_times <= 3:
            try:
                decode_data = requests.post(track_decode_api, data=data)
                if decode_data.content == b'500':  # 解码器返回错误值为 b'500'
                    logger.warning(
                        "解析服务错误！重启服务中... 第 %d 次" % retry_times)
                    time.sleep(3)
                    retry_times += 1
                    continue
                else:
                    tracks = json.loads(decode_data.content)
                    break
            except:
                logger.warning(
                    "解析失败！3 秒后重试第 %d 次..." % retry_times)
                time.sleep(3)
                retry_times += 1

        if retry_times > 3:
            logger.warning(
                "[SENSITIVE SHIP] 数据丢失 MMSI: %s" % mmsi)
            return
        logger.info('[SENSITIVE SHIP] MMSI %s 共 %d 个轨迹点 ' % (mmsi, len(tracks)))
        for track in tracks:
            last_time = track['utc'] * 1000
            if last_time > self.updated_btime:
                self.updated_btime = last_time
            lon = track['lon'] / 1000000
            lat = track['lat'] / 1000000
            speed = track['sog'] / 514
            trace = track['cog'] / 100
            head = track['hdg'] / 100
            item = VoyageTrackpointInfoItem()
            item['ID'] = str(uuid.uuid4())
            item['ShipMMSI'] = mmsi
            item['Trace'] = trace
            item['Speed'] = round(speed, 2)
            item['Head'] = head
            item['Longitude'] = lon
            item['Latitude'] = lat
            item['LastTime'] = last_time
            item['UpdateTime'] = int(time.time() * 1000)
            yield item

    def close(self, reason):
        if self.updated_btime > 0:
            self.redis_client.set("ShipSpiders:sensitive_latest_crawl_ts", str(self.updated_btime))
            logger.info("采集结束，下一次高敏轨迹采集开始时间为 {}".format(
                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(self.updated_btime / 1000))))

    def parse(self, response, **kwargs):
        rsp_obj = json.loads(response.text)
        slon = response.meta['slon']
        slat = response.meta['slat']
        elon = response.meta['elon']
        elat = response.meta['elat']
        current_area = 'Latitude: %.3f ~ %.3f, Longitude: %.3f ~ %.3f' % (
            slat / 1000000, elat / 1000000, slon / 1000000, elon / 1000000)
        if rsp_obj['status'] != 0:
            logger.warning('Response no data! %s' % response.text)
            return
        _count = rsp_obj['count']
        resolution = self.settings['MAP_RESOLUTION']
        if _count < 1524 or elon - slon < resolution * 1000000:  # 设置最小区域范围为resolution
            self.total_area += (elat - slat) * (elon - slon)
            self.total_progress = '%.2f%%' % float(
                self.total_area / self.global_area * 100)
            if rsp_obj['count'] == 0:
                logger.info('[Area Covered: %s] 0 ship found in area %s' % (
                    self.total_progress, current_area))
                return
            data = rsp_obj['data']
            self.total_ship_count += _count
            logger.info('[Area Covered: %s] %d ship(s) found in area %s' % (
                self.total_progress, _count, current_area))
            decode_api = self.settings['DECODE_SERVICE']
            retry_times = 1
            while retry_times <= 3:
                try:
                    decode_data = requests.post(decode_api, data=data)
                    if decode_data.content == b'500':  # 解码器返回错误值为 b'500'
                        logger.warning(
                            "Decode server error! Restarting server and retry for %d time(s) after 3 seconds..." % retry_times)
                        time.sleep(3)
                        retry_times += 1
                        continue
                    else:
                        ships = json.loads(decode_data.content)
                        break
                except:
                    logger.warning(
                        "Decode failure! Retrying for %d time(s) after 3 seconds..." % retry_times)
                    time.sleep(3)
                    retry_times += 1

            if retry_times > 3:
                logger.warning("%d ship data loss for area %s" %
                               (_count, current_area))
                return

            try:
                for _ship in ships:
                    update_voyage = True
                    if _ship['mmsi'] is None or str(_ship['mmsi']).encode() in self.sensitive_mmsi:
                        continue
                    meta_item = {
                        'ShipMMSI': _ship['mmsi'],
                        'Head': _ship['hdg'],
                        'Trace': _ship['cog'],
                        'Speed': _ship['sog'],  # 初始航速为节
                        'Longitude': _ship['lng'],
                        'Latitude': _ship['lat'],
                        'LastTime': _ship['lastdyn'] * 1000
                    }
                    if globe.is_land(meta_item['Latitude'], meta_item['Longitude']):
                        self.land_points += 1
                        if self.land_points % 1000 == 0:
                            logger.info('Filtered 1000 unusual points.')
                        continue
                    # if self.redisClient.hexists(name='ship_voyage', key=_ship['mmsi']):
                    if str(_ship['mmsi']).encode() in self.shipxy_voyage:
                        voyage_bstr = self.shipxy_voyage[str(_ship['mmsi']).encode()]
                        voyage = json.loads(voyage_bstr.decode())
                        current_timestamp_milli = int(round(time.time() * 1000))
                        update_voyage = False
                    else:
                        update_voyage = True

                    if update_voyage:
                        update_cookies_times = self.settings['UPDATE_COOKIES_AFTER_REQUEST_COUNT']
                        if self.voyage_request_times > update_cookies_times:
                            logger.info(
                                'Get %d Ship voyage info from shipxy' % update_cookies_times)
                            session_start_times = 1
                            while session_start_times <= 3:
                                try:
                                    self.driver.quit()
                                except:
                                    logger.info(
                                        'Old session quit error! Start new session anyway...')
                                try:
                                    self.driver.start_session(
                                        capabilities=DesiredCapabilities.FIREFOX)
                                    logger.info(
                                        'New session started! Refreshing shipxy.com webpage...')
                                    self.driver.get('http://www.shipxy.com/')
                                    break
                                except WebDriverException:
                                    logger.info(
                                        'Session start error! retrying start new session for %d time(s) atfer 3 seconds...' % session_start_times)
                                    session_start_times += 1
                                    time.sleep(3)

                            if session_start_times > 3:
                                logger.info(
                                    'Cannot start new session! Please check webdriver')
                            else:
                                logger.info('Updating cookies...')
                                time.sleep(1)
                                self.dict_cookie = self.driver.get_cookies()
                                self.voyage_request_times = 0

                        yield scrapy.FormRequest(
                            url='http://www.shipxy.com/ship/GetCurrvoyageNew',
                            callback=self.parse_voyage,
                            meta={'item': json.dumps(meta_item)},
                            formdata={'mmsi': str(_ship['mmsi'])},
                            cookies=self.dict_cookie)
                    else:
                        self.cached_voyage_read_times += 1
                        if self.cached_voyage_read_times >= 3000:
                            logger.info(
                                'Loaded 3000 Ship cached voyage info from REDIS')
                            self.cached_voyage_read_times = 0
                        item = VoyageTrackpointInfoItem()
                        item['ID'] = str(uuid.uuid4())
                        for _ in meta_item:
                            item[_] = meta_item[_]
                        for _ in voyage:
                            item[_] = voyage[_]
                        item['UpdateTime'] = current_timestamp_milli
                        yield item
            except:
                logger.error('Ship data process Error!')

        else:
            mlat = int((slat + elat) / 2)
            mlon = int((slon + elon) / 2)
            area = []
            area.append({'slon': slon, 'slat': slat,
                         'elon': mlon, 'elat': mlat})
            area.append({'slon': mlon, 'slat': slat,
                         'elon': elon, 'elat': mlat})
            area.append({'slon': slon, 'slat': mlat,
                         'elon': mlon, 'elat': elat})
            area.append({'slon': mlon, 'slat': mlat,
                         'elon': elon, 'elat': elat})
            url = 'http://www.shipxy.com/ship/getareashipssimple?level=11&area={slon},{slat},{elon},{elat}&enc=0&src=0&_t={time}'
            for sub_area in area:
                yield scrapy.Request(url=url.format(slat=sub_area['slat'],
                                                    slon=sub_area['slon'],
                                                    elat=sub_area['elat'],
                                                    elon=sub_area['elon'],
                                                    time=int(time.time() * 1000)),
                                     callback=self.parse,
                                     cookies=self.dict_cookie,
                                     meta={'slat': sub_area['slat'], 'slon': sub_area['slon'], 'elat': sub_area['elat'],
                                           'elon': sub_area['elon']})

    def parse_voyage(self, response):
        meta_item = json.loads(response.meta['item'])
        self.ship_mmsi_with_no_voyage = []
        self.voyage_request_times += 1
        item = VoyageTrackpointInfoItem()
        item['ID'] = str(uuid.uuid4())
        for _ in meta_item:
            item[_] = meta_item[_]
        voyage_obj = json.loads(response.text)
        if voyage_obj['status'] != 0:
            return
        voyages = []
        data_text = voyage_obj['data']
        try:
            data_text = data_text.replace('\"', '"')
            time_pattern = '%Y-%m-%d %H:%M:%S'
            current_timestamp_milli = int(round(time.time() * 1000))
            voyage = {}
            try:
                voyages.append(json.loads(data_text))
                for _voyage in voyages:
                    voyage['DepTime'] = to_unix_timestamp(time_pattern)  # 出发时间
                    voyage['DestPort'] = _voyage['destportname_en']  # 目的港口英文
                    voyage['DestTime'] = to_unix_timestamp(time_pattern)  # 预计抵达时间
                    voyage['FromPort'] = _voyage['depportname_en']  # 出发港口英文
                    voyage['Position'] = _voyage['position']  # 所在海域
                    voyage['Status'] = _voyage['navi_stat']  # 航行状态
                    for _ in voyage:
                        item[_] = voyage[_]
                    voyage['ExpireTimestamp'] = current_timestamp_milli + \
                                                (5 * 60 + random.randint(0, 5 * 60)) * 60 * 1000
                    self.new_voyage[meta_item['ShipMMSI']] = json.dumps(voyage, ensure_ascii=False)

            except:
                self.ship_mmsi_with_no_voyage.append(meta_item['ShipMMSI'])
                if len(self.ship_mmsi_with_no_voyage) > 10:
                    _list = ', '.join(self.ship_mmsi_with_no_voyage)
                    logger.info('Ship MMSI %s has no voyage data' % _list)
                    self.ship_mmsi_with_no_voyage = []
            finally:
                item['UpdateTime'] = current_timestamp_milli
                yield item
        except AttributeError:
            logger.debug('dict Attribute error')
            item['UpdateTime'] = int(round(time.time() * 1000))
            yield item
