import json
import logging
import random
from datetime import datetime

import scrapy

from FsyAgent.utils.http_utils import json_post


class FtsAgent(scrapy.Spider):
    name = 'fts_agent'
    start_urls = ['http://47.115.228.133:28080/static/data.json']
    insert_api = 'http://39.98.174.146:28089/RMWebApi/search/addInfoDoc'

    def __init__(self, params=None, *args, **kwargs):
        super(FtsAgent, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def parse(self, response, **kwargs):
        rsp_str = response.text
        dataset = json.loads(rsp_str)
        data = random.choice(dataset)['_source']
        current_time = datetime.now()
        formatted_time = current_time.strftime('%Y-%m-%d %H:%M:%S')
        insert_data = {'indexName': 'indeximos'}
        for key in data:
            if type(data[key]) == str and len(data[key]) > 0:
                insert_data[key] = data[key]
        insert_data['es_sitename'] = data['es_authors']
        insert_data['es_carriertype'] = 'news'
        insert_data['es_urltime'] = formatted_time
        insert_data['es_lasttime'] = formatted_time
        insert_data['es_loadtime'] = formatted_time
        result = json_post(url=self.insert_api, data=insert_data)
        logging.info(result.text)
        yield
