# -*- coding: utf-8 -*-
import json
import logging as logger
import re
import time
from urllib.parse import unquote

import scrapy

from MediaSpiders.items import MediaspidersItem
from MediaSpiders.utils.http_utils import http_post


class FlickrSpider(scrapy.Spider):
    name = "FlickrSpider"
    flicker_base_api = 'https://www.flickr.com'
    custom_settings = {
        'PROTO_MODULE_PATH': 'MediaSpiders.proto.Es_pb2',
        'PROTO_CLASS_NAME': 'EsSets',
        'PROTO_FIELD_NAME': 'Es',
        'PROTO_SAVE_FILE_NAME': 'public_info_data_',
        'IMAGES_STORE': r'/usr/local/temp_image/flickr',
        'IMAGES_RESULT_FIELD': 'es_urlimage',
        'FILES_STORE': r'/usr/local/videos',
        'FILES_RESULT_FIELD': 'es_video',
        'ZIP_FILE_NAME': 'image_data_publicinfo_',
        'FILE_ZIP_FILE_NAME': 'image_data_plane_',
        'DOWNLOAD_DELAY': 4,
        'ITEM_PIPELINES': {
            'scrapy.pipelines.images.ImagesPipeline': 2,
            'scrapy.pipelines.files.FilesPipeline': 1,
            'MediaSpiders.pipelines.ProtobufSavePipeline': 300,
        },
        'SPIDER_MIDDLEWARES': {
            'MediaSpiders.middlewares.DumpFilterSpiderMiddleware': 543
        }
    }

    def __init__(self, params=None, *args, **kwargs):
        super(FlickrSpider, self).__init__(*args, **kwargs)
        self.bloom_filter = None
        if params:
            json_params = json.loads(params)
            logger.info(json_params)
            if 'job_id' in json_params:
                self.job_id = json_params['job_id']

    def start_requests(self):
        self.simhash_filter_key = self.settings['FLICKR_SIMHASH_FILTER_KEY']
        account_query_api = self.settings['SOCIAL_USER_QUERY_ALL_API']
        account_query_api = account_query_api.format(sortBy="id", shuffleResult="true")
        post_data = {
            'userType': self.settings['FLICKR_USER_TYPE'],
            'userFlag': 0
        }
        res_body = json.loads(
            http_post(account_query_api, json.dumps(post_data), headers={"Content-Type": "application/json"}).text)
        if res_body['code'] != 200:
            logger.error('获取Flickr站点数据失败')
            return
        flickr_count = res_body['message']
        if int(flickr_count) == 0:
            logger.info('Flickr站点数据为0')
            return
        logger.info("获取{}个Flickr站点数据".format(flickr_count))
        for flickr_info in res_body['content']:
            flickr_url = '{0}/photos/{1}'.format(self.flicker_base_api, flickr_info['userUid'])
            yield scrapy.Request(flickr_url, callback=self.parse, meta={"flickr_info": flickr_info})

    def parse(self, response):
        flickr_info = response.request.meta["flickr_info"]
        self.filter_key = self.settings['FLICKR_FILTER_KEY']
        script_text = response.xpath("//script[@class='modelExport']/text()").get()
        target_regx = re.search(r'modelExport: {.*}', script_text)
        target_value = json.loads(target_regx.group().split(' ')[1])
        target_list = target_value['main']['photostream-models'][0]['data']['photoPageList']['data']['_data']
        for target in target_list:
            try:
                target_data = target['data']
                image_url = target_data['sizes']['data']['m']['data']['url']
                if image_url.startswith("//"):
                    image_url = "https:" + image_url
                item = MediaspidersItem()
                item['es_urlname'] = image_url
                item['es_sitename'] = flickr_info['userName']
                item['es_carriertype'] = 'image'
                item['es_loadtime'] = str(int(time.time() * 1000))
                item['es_catalog'] = flickr_info['keyWords']
                item['image_urls'] = [image_url]
                item['es_urltitle'] = unquote(target_data['title']) if 'title' in target_data else ''
                item['es_urltime'] = int(unquote(target_data['stats']['data']['datePosted'])) * 1000
                item['es_lasttime'] = int(unquote(target_data['stats']['data']['datePosted'])) * 1000
                item['es_urlcontent'] = unquote(target_data['description']) if 'description' in target_data else ''
                item['es_heat'] = target_data['engagement']['data']['viewCount'] if 'viewCount' in \
                                                                                    target_data['engagement'][
                                                                                        'data'] else 0
                # 加入判重因子
                response.request.meta['CUSTOM_FILTER_KEY'] = self.filter_key
                response.request.meta['CUSTOM_FILTER_VAL'] = image_url
                yield item
            except Exception as e:
                logger.error('图片信息解析错误，error：{}'.format(e))
