# -*- coding: utf-8 -*-

import base64
import importlib
import json
import logging
import os
import tarfile
import time
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
from typing import List

import pymysql
from kafka import KafkaProducer

from MediaSpiders.utils.http_utils import del_html_img
from MediaSpiders.utils.image_utils import filter_image


class ProtobufSavePipeline(object):
    def __init__(self):
        self.retention_hours = None
        self.module = None
        self.proto_class_name = None
        self.proto_field_name = None
        self.proto_save_file_path_name = None
        self.zip_file_path = None
        self.sets = None
        self.batch_save_number = None
        self.current_number = 0
        self.producer = None
        self.img_save_list = []
        self.attach_save_list = []

    def open_spider(self, spider):
        self.retention_hours = spider.settings['RETENTION_HOURS']
        module_path = spider.settings['PROTO_MODULE_PATH']
        self.proto_class_name = spider.settings['PROTO_CLASS_NAME']
        self.proto_field_name = spider.settings['PROTO_FIELD_NAME']
        self.proto_save_file_path_name = os.path.join(spider.settings['PROTO_SAVE_FILE_PATH'],
                                                      spider.settings['PROTO_SAVE_FILE_NAME'])
        self.module = importlib.import_module(module_path)
        self.sets = getattr(self.module, self.proto_class_name)()

        self.batch_save_number = spider.settings['BATCH_SAVE_SIZE']
        self.current_number = 0
        self.producer = KafkaProducer(bootstrap_servers=spider.settings['KAFKA_SERVER'])

        self.img_save_list = []
        self.attach_save_list = []

    def close_spider(self, spider):
        if not os.path.exists(spider.settings['PROTO_SAVE_FILE_PATH']):
            os.makedirs(spider.settings['PROTO_SAVE_FILE_PATH'])
        if len(getattr(self.sets, self.proto_field_name)) == 0:
            return
        to_path = os.path.join(spider.settings['FILE_TRANS_PATH'],
                               spider.settings['ZIP_FILE_NAME'] + str(int(time.time() * 1000)) + '.tar.gz')
        self.write_to_zip(self.img_save_list, to_path)
        self.img_save_list = []

        # 将数据发送至消息中心
        json_data = {
            'content': self.sets.SerializeToString().decode('ISO-8859-1'),
            'protoName': spider.settings['PROTO_SAVE_FILE_NAME'],
            'processQueue': spider.settings['KAFKA_PROCESS_QUEUE']
        }
        content = base64.b64encode(json.dumps(json_data).encode('utf-8'))
        self.producer.send(spider.settings['KAFKA_PROCESS_QUEUE'][0], content, partition=0)
        self.producer.flush()
        self.producer.close()

        self.del_file(spider.settings['PROTO_SAVE_FILE_PATH'])
        self.del_file(spider.settings['FILES_STORE'])
        self.del_file(spider.settings['IMAGES_STORE'])
        self.del_file(spider.settings['FILE_TRANS_PATH'])

    def process_item(self, item, spider):
        proto_data = getattr(self.sets,
                             self.proto_field_name).add()  # 根据约定的proto文件格式，初始化带有需要特定字段的数据对象protoData与sets进行绑定
        for field in item.fields:
            if field in item and item[field] != '' and hasattr(proto_data, field):  # 只有item的字段与protoData字段对应，且值不为空的可以赋值
                if isinstance(item[field], List):  # protoData各个字段的值都应是字符串。对于字段的值是一个序列的，进行json.dumps序列化操作，使之转化为字符串
                    setattr(proto_data, field, json.dumps(item[field]))
                    # for t in item[field]:
                    #     getattr(protoData, field).append(t)
                else:
                    setattr(proto_data, field, str(item[field]))
                if field == spider.settings['IMAGES_RESULT_FIELD']:
                    # 对图片url字段进行处理，由于image_pipeline优先级比本pipeline高，图片实际上已经下载完成。文件同理
                    item['es_urlcontent'] = item['es_urlcontent'].replace('&amp;', '&')
                    img_filter_list = []
                    for img in item[field]:
                        filter_flag, img_url = filter_image(spider.settings['IMAGES_STORE'], img, item)
                        if filter_flag:
                            img_filter_list.append(img_url)
                            continue
                        self.img_save_list.append(img_url)
                        # 将原文中图片原始链接替换为我方图片存储服务其上的相对路径
                        item['es_urlcontent'] = item['es_urlcontent'].replace(img['url'], '/' + img['path'])
                    # 删除item中已过滤的图片url
                    item[field] = [img for img in item[field] if img['url'] not in img_filter_list]
                    # 删除正文中已过滤图片的img标签
                    item['es_urlcontent'] = del_html_img(item['es_urlcontent'], img_filter_list)
                    item['es_urlcontent'] = item['es_urlcontent'].replace('data-src=', 'src=')
                    setattr(proto_data, 'es_urlcontent', str(item['es_urlcontent']))
                    setattr(proto_data, 'es_urlimage', str(item['es_urlimage']))
                if field == spider.settings['FILES_RESULT_FIELD']:
                    for attach in item[field]:
                        self.attach_save_list.append(os.path.join(spider.settings['FILES_STORE'], attach['path']))
        # 批次保存采集结果
        if self.current_number >= self.batch_save_number:
            if not os.path.exists(spider.settings['PROTO_SAVE_FILE_PATH']):
                os.makedirs(spider.settings['PROTO_SAVE_FILE_PATH'])
            to_path = os.path.join(spider.settings['FILE_TRANS_PATH'],
                                   spider.settings['ZIP_FILE_NAME'] + str(int(time.time() * 1000)) + '.tar.gz')
            self.write_to_zip(self.img_save_list, to_path)  # 调用write2zip将图片/文件打包成压缩包
            self.img_save_list = []

            # 附件压缩转移到传输服务监听目录
            to_path = os.path.join(spider.settings['FILE_TRANS_PATH'],
                                   spider.settings['FILE_ZIP_FILE_NAME'] + str(int(time.time() * 1000)) + '.tar.gz')
            self.write_to_zip(self.attach_save_list, to_path)
            self.attach_save_list = []
            # 将数据发送至消息中心
            json_data = {
                'content': self.sets.SerializeToString().decode('ISO-8859-1'),  # 将sets内容序列化并使用加密编码生成jsonData
                'protoName': spider.settings['PROTO_SAVE_FILE_NAME'],
                'processQueue': spider.settings['KAFKA_PROCESS_QUEUE']
            }
            content = base64.b64encode(json.dumps(json_data).encode('utf-8'))  # 将jsonData对象序列化并加密编码生成content
            self.producer.send(spider.settings['KAFKA_PROCESS_QUEUE'][0], content,
                               partition=0)  # 将content发送至KAFKA_PROCESS_QUEUE定义的第一个处理单元中
            # 发送完毕，重新初始化
            self.producer.flush()
            self.current_number = 0
            self.sets = getattr(self.module, self.proto_class_name)()
        self.current_number += 1
        return item

    def write_to_zip(self, file_list, to_path):
        if len(file_list) == 0:
            return
        with tarfile.open(to_path, 'w:gz') as zipFile:
            for filePath in file_list:
                try:
                    if not os.path.exists(filePath):
                        continue
                    arc_name = "full" + filePath[filePath.rindex('/'):]  # 文件相对路径
                    zipFile.add(filePath, arc_name)
                except:
                    pass
        for filePath in file_list:
            if not os.path.exists(filePath):
                continue
            os.remove(filePath)
        # 发送压缩包至kafka
        self.send_file_info_to_kafka(to_path)

    def send_file_info_to_kafka(self, file_path):
        last_index = file_path.rfind('/')
        if last_index < file_path.rfind('\\'):
            last_index = file_path.rfind('\\')
        file_name = file_path[last_index + 1:]
        HOST_IP = os.getenv('HOST_IP')
        file_url = f'http://{HOST_IP}:28086/{file_name}'
        file_info = {
            'fileName': file_name,
            'url': file_url
        }
        content = base64.b64encode(json.dumps(file_info).encode('utf-8'))
        self.producer.send('stream-file-dl', content, partition=0)
        self.producer.flush()

    def del_file(self, path):
        for root, dirs, files in os.walk(path, topdown=False):
            for file in files:
                filePath = os.path.join(root, file)
                st_info = os.stat(filePath)
                if time.time() - st_info.st_mtime > self.retention_hours * 60 * 60:
                    os.remove(filePath)


class HotSearchSaveToMySQL(object):

    def __init__(self):
        self.db = None
        self.cursor = None

    def open_spider(self, spider):
        self.db = pymysql.connect(host='47.113.231.200', port=28089, user='root', passwd='passok123A',
                                  db='hot_search', charset='utf8mb4')
        self.cursor = self.db.cursor()

    def close_spider(self, spider):
        self.db.close()

    def process_item(self, item, spider):
        cols = []
        vals = []
        for field in item.fields:
            if field in item and item[field] and item[field] != '':
                cols.append(field)
                v = item[field]
                if type(v) == int:
                    vals.append(f"{v}")
                else:
                    v = v.replace("'", "’")
                    vals.append(f"'{v}'")
        SQL_INSERT = f"INSERT INTO hot_search.hot_search_board ({', '.join(cols)}) VALUES ({', '.join(vals)})"
        try:
            self.cursor.execute(SQL_INSERT)
            self.db.commit()
        except pymysql.err.IntegrityError as ie:
            logging.debug(f"有重复内容：{SQL_INSERT}")
            logging.debug(repr(ie))
        except pymysql.err.DataError as de:
            logging.error(repr(de))
        return item


class TelegramDataSaveToMySQL(object):
    def __init__(self):
        self.db = None
        self.cursor = None

    def open_spider(self, spider):
        self.db = pymysql.connect(host='39.101.194.63', port=23306, user='root', passwd='passok123A',
                                  db='social_user_info', charset='utf8mb4')
        self.cursor = self.db.cursor()

    def close_spider(self, spider):
        self.db.close()

    def process_item(self, item, spider):
        class_name = item.__class__.__name__
        table_name = ''
        if class_name == 'TelegramGroup':
            table_name = 'tg_group'
        if class_name == 'TelegramMessage':
            table_name = 'tg_message'
        if class_name == 'TelegramMember':
            table_name = 'tg_member'
        cols = []
        vals = []
        for field in item.fields:
            if field in item and item[field] and item[field] != '':
                cols.append(field)
                v = item[field]
                if type(v) == int:
                    vals.append(f"{v}")
                else:
                    v = v.replace("'", "’")
                    vals.append(f"'{v}'")
        SQL_INSERT = f"INSERT INTO social_user_info.{table_name} ({', '.join(cols)}) VALUES ({', '.join(vals)})"
        try:
            self.cursor.execute(SQL_INSERT)
            self.db.commit()
        except pymysql.err.IntegrityError as ie:
            logging.debug(f"有重复内容：{SQL_INSERT}")
            logging.debug(repr(ie))
        except pymysql.err.DataError as de:
            logging.error(repr(de))
        return item
