import scrapy
import json
from MediaSpiders.utils.hot_search_json_parser import parse_weibo_response, parse_toutiao_response
from MediaSpiders.items import MediaspidersItem


class HotSearchSpider(scrapy.Spider):
    name = "hot_search_spider"

    custom_settings = {
        'PROTO_MODULE_PATH': 'MediaSpiders.proto.Es_pb2',
        'PROTO_CLASS_NAME': 'EsSets',
        'PROTO_FIELD_NAME': 'Es',
        'PROTO_SAVE_FILE_NAME': 'public_info_data_',
        'IMAGES_STORE': r'/usr/local/temp_image/twitter',
        'IMAGES_RESULT_FIELD': 'es_urlimage',
        'FILES_STORE': r'/usr/local/videos',
        'FILES_RESULT_FIELD': 'es_video',
        'ZIP_FILE_NAME': 'image_data_publicinfo_',
        'FILE_ZIP_FILE_NAME': 'image_data_plane_',
        'ITEM_PIPELINES': {
            'MediaSpiders.pipelines.ProtobufSavePipeline': 300,
            # 'MediaSpiders.pipelines.HotSearchSaveToMySQL': 300
        },
        'SPIDER_MIDDLEWARES': {},
        'DOWNLOADER_MIDDLEWARES': {},
        'BATCH_SAVE_SIZE': 50
    }

    start_urls = [
        'https://weibo.com/ajax/side/hotSearch',
        'https://www.toutiao.com/hot-event/hot-board/?origin=toutiao_pc'
    ]

    def __init__(self, params=None, *args, **kwargs):
        super(HotSearchSpider, self).__init__(*args, **kwargs)
        if params:
            json_params = json.loads(params)
            if 'job_id' in json_params:
                self.job_id = json_params['job_id']

    def parse(self, response):
        result_array = []
        if 'weibo.com' in response.url:
            result_array = parse_weibo_response(response.text)
        elif 'toutiao.com' in response.url:
            result_array = parse_toutiao_response(response.text)

        for line in result_array:
            hot_search_item = MediaspidersItem()
            hot_search_item['es_carriertype'] = 'hot_search'
            hot_search_item['es_sid'] = line['id']
            hot_search_item['es_hkey'] = line['hot_id']
            hot_search_item['es_urltitle'] = line['hot_word']
            hot_search_item['es_urlcontent'] = line['hot_word']
            hot_search_item['es_heat'] = line['hot_value']
            hot_search_item['es_catalog'] = line['category']
            hot_search_item['es_simrank'] = line['realtime_rank']
            hot_search_item['es_sitename'] = line['platform']
            hot_search_item['es_urltime'] = line['onboard_time']
            hot_search_item['es_lasttime'] = line['crawl_time']
            hot_search_item['es_urlname'] = line['fake_url']
            yield hot_search_item
