# -*- coding: utf-8 -*-
import json
import re

import scrapy
import validators
from scrapy_redis.spiders import RedisSpider

from WebsiteSpider.scrapy_selenium import SeleniumRequest
from WebsiteSpider.utils.http_utils import build_url
from WebsiteSpider.utils.parser_utils import parse_item_from_response


class WebsiteInfoCommonSpider(RedisSpider):
    name = 'website_info_common'

    def __init__(self, params=None, *args, **kwargs):
        super(WebsiteInfoCommonSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        self.name = 'WebSite_' + json_params['clusterName']
        self.redis_client = None
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def get_job_id(self):
        return self.job_id

    def parse(self, response, **kwargs):
        content_type = response.headers.get('Content-Type')
        if content_type is not None and content_type.decode().find('text/html') < 0:
            return
        current_url = response.url
        parse_rule = response.request.meta['parse_rule']
        allowed_domains = parse_rule['allowDomain'].split(';')
        field_mappings = parse_rule['fieldMappings']
        # 根据url特征判断是否为内容页，若是则解析文本内容
        detail_page_reg = parse_rule['detailPageReg']
        if detail_page_reg == "" or re.search(detail_page_reg, response.url) is not None:
            yield_flag, webpage_item = parse_item_from_response(response, parse_rule)
            if yield_flag:
                yield webpage_item

        # 从页面中解析出新的url并进行爬取
        parse_rule['current_depth'] += 1
        if parse_rule['current_depth'] > int(parse_rule['depth']):
            return
        a_link_element = response.xpath("//a")
        for e in a_link_element:
            # 根据内容页正则判断使用哪一套解析模板
            url = e.xpath("@href").get()
            title = e.xpath(".//text()").get()
            if 'urlPrefix' in field_mappings:
                try:
                    url = field_mappings['urlPrefix'] + url
                except:
                    pass
            url = build_url(current_url, allowed_domains, url)
            if not validators.url(url):
                continue
            pattern = 'mailto|\.pdf|\.jpg|\.gif|\.doc|\.xls|\.print|\.mp3|\.mp4'
            if re.search(pattern, url):
                continue
            meta_data = {'parse_rule': parse_rule, 'CUSTOM_FILTER_KEY': 'URL_Filter:' + parse_rule['url'],
                         'CUSTOM_FILTER_VAL': url,
                         'title': title}
            if parse_rule['isDynamicSite']:
                yield SeleniumRequest(url=url, callback=self.parse, meta=meta_data)
            else:
                yield scrapy.Request(url, callback=self.parse, meta=meta_data)

    def closed(self, reason):
        if self.server is not None:
            self.server.close()
