import json
import logging
import time

import scrapy
import validators
from scrapy.loader import ItemLoader

from CustomWebsite.items import WebsiteSpiderItem
from CustomWebsite.utils.date_utils import get_time_stamp
from CustomWebsite.utils.http_utils import build_url
from CustomWebsite.utils.string_utils import get_str_md5


class WebsiteSpider(scrapy.Spider):
    name = 'blog_zhongjianjun'

    start_urls = [
        'https://nakamahitoshi.ti-da.net'
    ]

    def __init__(self, params=None, *args, **kwargs):
        super(WebsiteSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']

    def parse(self, response, **kwargs):
        lines = response.xpath("//div[@class='blog']/div[@class='blogbody']")
        line_num = 0
        for line in lines:
            line_num += 1
            content = "".join(line.xpath("./div[@class='main']//text()").extract())
            content = content.strip()
            title = "".join(line.xpath("./h3[@class='title']//text()").extract())
            content_url = self.start_urls[0] + "".join(line.xpath("./h3[@class='title']/a/@href").extract())
            date_str = "".join(response.xpath(
                f"//div[@id='content']/div[@class='blog']/h2[@class='date'][{line_num}]/text()").extract())
            time_str = "".join(line.xpath("./div[@class='posted']/a[@class='aposted'][1]/text()").extract())
            date_time_str = date_str + " " + time_str
            date_time_ts = get_time_stamp(date_time_str, None)
            image_urls = line.xpath("./div[@class='main']/img[@class='lazy']/@data-original").extract()
            images = []
            if len(image_urls) > 0:
                images = [f"https:{_}" for _ in image_urls]
            page_item = WebsiteSpiderItem()
            page_item['es_urltitle'] = title
            page_item['es_urlcontent'] = content
            page_item['es_urltime'] = date_time_ts
            page_item['es_urlname'] = content_url
            page_item['es_urlimage'] = images
            page_item['es_lasttime'] = page_item['es_urltime']
            page_item['es_sitename'] = 'twitter'
            page_item['es_srcname'] = 'twitter'
            page_item['es_carriertype'] = 'media'
            page_item['es_loadtime'] = str(int(time.time() * 1000))
            page_item['es_lang'] = 'JPN'
            page_item['es_doclength'] = len(page_item['es_urlcontent'])
            page_item['es_commentcount'] = 0
            page_item['es_forwardcount'] = 0
            page_item['es_likecount'] = 0
            page_item['es_authors'] = 'nakamahitoshi'
            page_item['es_extname'] = '仲間均'
            page_item['es_userid'] = '1167051552961392640'
            page_item['es_district'] = '沖縄県石垣島'
            page_item['es_hkey'] = get_str_md5(content_url)
            response.request.meta['CUSTOM_FILTER_KEY'] = 'URL_Filter:Custom:' + self.name
            response.request.meta['CUSTOM_FILTER_VAL'] = page_item['es_urlname']
            yield page_item
            # logging.info(page_item)
