# -*- coding: utf-8 -*-
import json
import logging as logger
import time

import redis
import scrapy
import validators
from scrapy.http import HtmlResponse
from scrapy.selector import Selector
from scrapy.utils.project import get_project_settings

from CustomWebsite.items import WebsiteSpiderItem
from CustomWebsite.utils.date_utils import get_time_stamp
from CustomWebsite.utils.http_utils import build_url


class WebsiteSpider(scrapy.Spider):
    name = 'rimpeace'
    custom_settings = {
        'FEED_EXPORT_ENCODING': 'Shift-JIS'
    }
    settings = get_project_settings()

    def __init__(self, params=None, *args, **kwargs):
        super(WebsiteSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def start_requests(self):
        yield scrapy.Request(url='http://www.rimpeace.or.jp/', callback=self.parse_homepage, dont_filter=True)

    def parse_homepage(self, response):
        content_area = response.xpath('//table/tbody/tr/td/ul/li')
        count = 0
        for content in content_area:
            try:
                a_tags = content.xpath('./a')
                if len(a_tags) <= 0:
                    continue
                elif len(a_tags) == 1:
                    urltitle = ''.join(content.xpath('.//text()').extract()).strip()
                    urlname = content.xpath('./a/@href').extract()[0].strip()
                    count += 1
                    if count > 20:
                        break
                    yield scrapy.Request(url=urlname, callback=self.parse_detail, dont_filter=True,
                                         meta={'urlname': urlname, 'urltitle': urltitle,
                                               'CUSTOM_FILTER_KEY': 'URL_Filter:Custom:' + self.name,
                                               'CUSTOM_FILTER_VAL': urlname
                                               })
                else:
                    for a in a_tags:
                        urltitle = ''.join(a.xpath('.//text()').extract()).strip()
                        urlname = a.xpath('./@href').extract()[0].strip()
                        count += 1
                        if count > 20:
                            break
                        yield scrapy.Request(url=urlname, callback=self.parse_detail, dont_filter=True,
                                             meta={'urlname': urlname, 'urltitle': urltitle,
                                                   'CUSTOM_FILTER_KEY': 'URL_Filter:Custom:' + self.name,
                                                   'CUSTOM_FILTER_VAL': urlname
                                                   })
            except:
                logger.warning("skip illegal urltitle")

    def parse_detail(self, response):
        rsp_text = ''
        try:
            rsp_text = response.body.decode('Shift-JIS')
        except:
            logger.warning("Cannot decode url '%s' for Shift-JIS" % response.url)
        result = HtmlResponse(rsp_text, body=rsp_text, encoding='utf-8')
        selector = Selector(response=result)
        urlname = response.meta['urlname']
        urltitle = response.meta['urltitle']
        logger.info("%s - %s" % (urltitle, urlname))
        urlcontent = ''.join(selector.xpath('//body/p').extract())
        raw_img_urls = selector.xpath('//img/@src').extract()
        try:
            raw_urltime = selector.xpath('//tt/text()[1]').extract()[0]
        except:
            logger.warning("urltime error! URL: %s" % urlname)
            return
        img_urls = []
        for temp_url in raw_img_urls:
            full_url = build_url(urlname, 'rimpeace.or.jp', temp_url)
            if validators.url(full_url):
                img_urls.append(full_url)
        urltime = get_time_stamp(raw_urltime, None)
        item = WebsiteSpiderItem()
        item['es_urltitle'] = urltitle
        item['es_urlcontent'] = urlcontent
        item['es_urltime'] = urltime
        item['es_lasttime'] = urltime
        item['es_urlimage'] = img_urls
        item['es_urlname'] = urlname
        item['es_sitename'] = '追踪驻日美军'
        item['es_catalog'] = '武装力量'
        item['es_collection'] = '新闻'
        item['es_lang'] = 'JPN'
        item['es_srcname'] = 'http://www.rimpeace.or.jp/'
        item['es_startid'] = '133'
        item['es_loadtime'] = str(int(time.time() * 1000))
        item['es_carriertype'] = 'news'
        item['es_doclength'] = len(item['es_urlcontent'])
        response.request.meta['CUSTOM_FILTER_KEY'] = 'URL_Filter:Custom:' + self.name
        response.request.meta['CUSTOM_FILTER_VAL'] = item['es_urlname']
        yield item
