# -*- coding: utf-8 -*-
import json
import logging as logger
import time

from scrapy_redis.spiders import RedisSpider

from WebsiteSpider.items import WebsiteSpiderItem
from WebsiteSpider.utils.date_utils import get_time_stamp
from WebsiteSpider.utils.date_utils import transfer_time_zone
from WebsiteSpider.utils.http_utils import build_url


class WebsiteInfoCommonSpider(RedisSpider):
    name = 'Website_report_list'
    custom_settings = {
        'DOWNLOADER_MIDDLEWARES': {
            'WebsiteSpider.middlewares.InitReportListDownloaderMiddleware': 299,
            'WebsiteSpider.middlewares.UserAgentDownloaderMiddleware': 799,
            'WebsiteSpider.scrapy_selenium.SeleniumMiddleware': 800
        },
        'SPIDER_MIDDLEWARES': {
            'WebsiteSpider.middlewares.DumpFilterSpiderMiddleware': 544
        }

    }

    def __init__(self, params=None, *args, **kwargs):
        super(WebsiteInfoCommonSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        self.redis_client = None
        self.yield_reports = True
        if 'yield_reports' in json_params:
            self.yield_reports = json_params['yield_reports']
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def get_job_id(self):
        return self.job_id

    def parse(self, response, **kwargs):
        content_type = response.headers.get('Content-Type')
        if content_type is not None and content_type.decode().find('text/html') < 0:
            return
        parse_rule = response.request.meta['parse_rule']
        rules = parse_rule['fieldMappings']
        domain = parse_rule['allowDomain'].split(';')
        time_zone = int(float(parse_rule['timeZone']))
        report_list_area = response.xpath(rules['report_list_area'])
        for field in ['es_urltitle', 'es_attachment', 'es_urlcontent']:
            field_rule = rules[field]
            if field_rule.split('/')[-1] != 'text()' and '@' not in field_rule.split('/')[-1]:
                rules[field] = rules[field] + "//text()"
        if len(report_list_area) > 0:
            logger.info("当前站点“{}”的报告列表一共有{}行".format(parse_rule['siteName'], len(report_list_area)))
            for line in report_list_area:
                try:
                    title_info = line.xpath(rules['es_urltitle']).extract()
                    link_info = line.xpath(rules['es_attachment']).extract()
                    date_info = line.xpath(rules['es_urltime']).extract()
                    content_info = line.xpath(rules['es_urlcontent']).extract()
                    if link_info is None or len(link_info) <= 0:
                        continue
                    pdf_link = link_info[0]
                    title_date = date_info[0]
                    title_name = "".join(title_info).strip()
                    pdf_link = build_url(response.url, domain, pdf_link)
                    file_urls = [pdf_link]
                    time_parse_rule = None
                    if 'dateReg' in rules:
                        time_parse_rule = {
                            rules['dateReg']: [rules['dateFormate']]
                        }
                    urltime = transfer_time_zone(get_time_stamp(title_date, time_parse_rule), time_zone)
                    report_item = WebsiteSpiderItem()
                    report_item['es_urltitle'] = title_name
                    report_item['es_urlcontent'] = "".join(content_info).strip()
                    report_item['es_attachment'] = file_urls
                    report_item['es_attention'] = title_name
                    report_item['es_urltime'] = urltime
                    report_item['es_lasttime'] = urltime
                    report_item['es_urlname'] = pdf_link
                    report_item['es_sitename'] = parse_rule['siteName']
                    report_item['es_srcname'] = parse_rule['url']
                    report_item['es_startid'] = parse_rule['id']
                    report_item['es_loadtime'] = str(int(time.time() * 1000))
                    report_item['es_carriertype'] = parse_rule['carrierType']
                    report_item['es_doclength'] = len(report_item['es_urlcontent'])
                    if 'siteType' in parse_rule and parse_rule['siteType']:
                        report_item['es_catalog'] = parse_rule['siteType']
                    if 'resourceType' in parse_rule and parse_rule['resourceType']:
                        report_item['es_collection'] = parse_rule['resourceType']
                    if 'region' in parse_rule and parse_rule['region']:
                        report_item['es_lang'] = parse_rule['region']
                    logger.info("报告《{}》解析成功，下载链接是{}".format(title_name, pdf_link))
                    response.request.meta['CUSTOM_FILTER_KEY'] = 'URL_Filter:' + parse_rule['url']
                    response.request.meta['CUSTOM_FILTER_VAL'] = pdf_link
                    yield report_item
                except Exception as e:
                    logger.error(repr(e))
                    continue

    def closed(self, reason):
        if self.redis_client is not None:
            self.redis_client.close()
