# -*- coding: utf-8 -*-
import json
import logging as logger
import time

import validators
from scrapy.loader import ItemLoader

from CustomWebsite.items import WebsiteSpiderItem
from CustomWebsite.utils.date_utils import transfer_time_zone, get_time_stamp
from CustomWebsite.utils.http_utils import filter_html_tags, build_url


def parse_item_from_response(response, parse_rule):
    current_url = response.url
    allowed_domains = parse_rule['allowDomain'].split(';')
    mapping = parse_rule['fieldMappings']
    time_zone = float(parse_rule['timeZone'])
    webpage_item = WebsiteSpiderItem()
    item_loader = ItemLoader(item=webpage_item, response=response)
    # 根据内容页正则判断使用哪一套解析模板
    for field in webpage_item.fields:
        if field in mapping:
            if field == 'es_urlimage':
                # item_loader.add_xpath(field, mapping[field] + '/@src')
                # 图片的真实链接存储在其他属性中，不在src属性中,需要在规则中添加@，否则规则中不需要具体到@
                if '@' in str(mapping[field]).split('/')[-1]:
                    item_loader.add_xpath(field, mapping[field])
                else:
                    item_loader.add_xpath(field, mapping[field] + '/@src')
            elif field not in ['es_urlcontent']:
                rule_suffix = mapping[field].split('/')[-1]
                # 如果配置规则中已添加/text()或者使用@提取了属性值，则不加//text()
                if rule_suffix[:6] == 'text()' or rule_suffix[0] == '@':
                    item_loader.add_xpath(field, mapping[field])
                else:
                    item_loader.add_xpath(field, mapping[field] + '//text()')
            else:
                item_loader.add_xpath(field, mapping[field])
    webpage_item = item_loader.load_item()
    for field in webpage_item:
        if webpage_item[field] is None:
            continue
        if field == 'es_urlcontent':
            src = '@src'  # 图片标签默认使用的链接存储在src属性中
            if 'es_urlimage' in mapping:
                # new_content_xpath = mapping[field] + ' | ' + mapping['es_urlimage']
                if '@' in str(mapping['es_urlimage']).split('/')[-1]:  # 图片的真实链接存储在其他属性中，不在src属性中
                    src = str(mapping['es_urlimage']).split('/')[-1]
                    new_content_xpath = mapping[field] + ' | ' + str(mapping['es_urlimage']).replace(
                        '/' + src, '')
                else:
                    new_content_xpath = mapping[field] + ' | ' + mapping['es_urlimage']
            else:
                new_content_xpath = mapping[field]
            element_list = response.xpath(new_content_xpath)
            page_content = ''
            for element in element_list:
                # 保留图片在原始网页中位置
                if element.xpath(src):
                    img_src = element.xpath(src).get()
                    if '?' in img_src:
                        img_src = img_src.split('?')[0]
                        logger.info(img_src)
                    img_src = build_url(current_url, allowed_domains, img_src)
                    if not validators.url(img_src):
                        continue
                    # 避免插入重复图片
                    if img_src not in str(webpage_item['es_urlcontent']):
                        page_content += "<img src=%s>" % img_src
                    page_content += '<br/>\r\n'
                else:
                    text_element = element.getall()
                    for t in text_element:
                        t = t.strip('\r\n')
                        if t.isspace():
                            continue
                        page_content += t
                        page_content += '<br/>\r\n'
            page_content = filter_html_tags(page_content)
            webpage_item[field] = page_content
        elif field == 'es_urlimage' or field == 'es_attachment':
            temp_urls = webpage_item[field]
            webpage_item[field] = []
            for temp_url in temp_urls:
                if field == 'es_urlimage' and '?' in temp_url:
                    temp_url = temp_url.split('?')[0]
                full_url = build_url(current_url, allowed_domains, temp_url)
                if validators.url(full_url):
                    webpage_item[field].append(full_url)
            logger.info(webpage_item[field])
        else:
            webpage_item[field] = ''.join(webpage_item[field])

    webpage_item['es_sitename'] = parse_rule['siteName']
    webpage_item['es_srcname'] = parse_rule['url']
    webpage_item['es_startid'] = parse_rule['id']
    webpage_item['es_tags'] = ''
    if 'siteType' in parse_rule and parse_rule['siteType']:
        webpage_item['es_catalog'] = parse_rule['siteType']
    if 'resourceType' in parse_rule and parse_rule['resourceType']:
        webpage_item['es_collection'] = parse_rule['resourceType']
    webpage_item['es_urlname'] = response.url
    logger.info("urlname: %s" % webpage_item['es_urlname'])
    webpage_item['es_loadtime'] = str(int(time.time() * 1000))
    webpage_item['es_carriertype'] = parse_rule['carrierType']
    webpage_item['es_urltopic'] = json.dumps(parse_rule['storageOption'])
    webpage_item['es_warning'] = json.dumps(parse_rule['keywordFilter'])
    if 'es_urlcontent' in webpage_item:
        webpage_item['es_doclength'] = len(webpage_item['es_urlcontent'])
    else:
        webpage_item['es_doclength'] = 0
    # 根据正则表达式对时间进行处理，转换成时间戳
    try:
        logger.info("urltime: %s" % webpage_item['es_urltime'])
    except KeyError:
        logger.info('时间解析失败，当前页面url: %s' % response.url)
    time_parse_rule = None
    if 'dateReg' in mapping:
        time_parse_rule = {
            mapping['dateReg']: [mapping['dateFormate']]
        }
    for time_field in ['es_lasttime', 'es_urltime']:
        if time_field in webpage_item and len(webpage_item[time_field]) > 0:
            webpage_item[time_field] = transfer_time_zone(
                get_time_stamp(webpage_item[time_field], time_parse_rule), time_zone)
    try:
        logger.info("es_urltitle: %s" % webpage_item['es_urltitle'])
    except KeyError:
        logger.info('标题解析失败，当前页面url: %s' % response.url)
    try:
        logger.info("es_urlcontent: %s" % webpage_item['es_urlcontent'][:50] + '......')
    except KeyError:
        logger.info('内容解析失败，当前页面url: %s' % response.url)

    title_is_not_empty = 'es_urltitle' in webpage_item and webpage_item['es_urltitle'] != ''
    content_is_not_empty = 'es_urlcontent' in webpage_item and webpage_item['es_urlcontent'] != ''
    doc_length_is_valid = 'docLength' not in parse_rule or webpage_item['es_doclength'] >= parse_rule['docLength']
    time_is_valid = 'es_urltime' in webpage_item and webpage_item['es_urltime'] != '' and webpage_item[
        'es_urltime'] != '0' and 0 < int(webpage_item['es_urltime']) < int(time.time() * 1000)
    filter_VIP_content = 'es_positiveWords' in webpage_item and mapping['filterVIPWords'] in webpage_item[
        'es_positiveWords']

    yield_flag = title_is_not_empty and content_is_not_empty and doc_length_is_valid and time_is_valid and not filter_VIP_content
    if not yield_flag:
        logger.info('URL为' + response.url + '的内容未解析，该页面可能不是内容页，也可能由下列解析规则问题导致')
        if not title_is_not_empty:
            logger.info('标题为空，解析规则是：' + mapping['es_urltitle'])
        if not content_is_not_empty:
            logger.info('内容为空：解析规则是：' + mapping['es_urlcontent'])
        if not doc_length_is_valid:
            logger.info('正文长度不满足要求，最小长度为：' + parse_rule['docLength'])
        if not time_is_valid:
            logger.info('时间无法解析，解析规则是：' + mapping['es_urltime'])
        if filter_VIP_content:
            logger.info('当前内容是VIP文章，并不完整，已经过滤。')
    return yield_flag, webpage_item
