import json
import time

import pymysql
import scrapy
from lxml import etree
from scrapy.utils.project import get_project_settings

from WikipediaSpiders.items import WikiSpidersItem
from WikipediaSpiders.utils.parse_utils import ZhParser, JpParser, EnParser
from WikipediaSpiders.utils.string_utils import get_str_md5


class WikiSpider(scrapy.Spider):
    name = 'wiki_spider'
    settings = get_project_settings()

    def __init__(self, params=None, *args, **kwargs):
        super(WikiSpider, self).__init__(*args, **kwargs)
        json_params = json.loads(params)
        if 'job_id' in json_params:
            self.job_id = json_params['job_id']
        self.mysql_db = pymysql.connect(host=self.settings['MYSQL_DB_HOST'], port=self.settings['MYSQL_DB_PORT'],
                                        user=self.settings['MYSQL_DB_USER'], passwd=self.settings['MYSQL_DB_PASSWD'],
                                        db=self.settings['MYSQL_DB_SCHEMA'], charset='utf8mb4')
        self.cursor = self.mysql_db.cursor()

    def start_requests(self):
        SQL_SELECT = "SELECT id, title, target_type, page_url, page_language, page_last_update " \
                     "FROM oscm.wiki_page_target WHERE del_flag = 0 ORDER BY page_crawl_time LIMIT 5;"
        self.cursor.execute(SQL_SELECT)
        target_url_db_data = self.cursor.fetchall()
        for line in target_url_db_data:
            line_url = line[3]
            if line_url.startswith('https://zh.wikipedia.org/wiki'):
                line_url.replace('/wiki/', '/zh-cn/')
            meta_info = {
                'id': line[0],
                'title': line[1],
                'target_type': line[2],
                'page_url': line_url,
                'page_language': line[4],
                'page_last_update': line[5],
                'proxy': 'http://127.0.0.1:10809'
            }
            yield scrapy.Request(url=meta_info['page_url'], meta=meta_info, callback=self.parse)

    def parse(self, response, **kwargs):
        page_language = response.request.meta['page_language']
        page_url = response.request.meta['page_url']
        target_type = response.request.meta['target_type']
        if page_language == 'en':
            page_parser = EnParser()
        elif page_language == 'jp':
            page_parser = JpParser()
        elif page_language == 'zh':
            page_parser = ZhParser()
        else:
            return
        selector = etree.HTML(response.text)
        title = selector.xpath("//h1[@id='firstHeading']//text()")[0]
        photo, infobox_dict = page_parser.parse_infobox_info(selector)
        toc = page_parser.parse_toc(selector)
        contents = page_parser.parse_contents(selector)
        last_modified = page_parser.parse_modified_time(selector)
        current_date_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        image_urls = ["https:" + _ for _ in photo]
        for content in contents:
            if 'images' in content:
                for image in content['images']:
                    image_urls.append('https:' + image)
        page_item = WikiSpidersItem()
        page_item['es_urltitle'] = str(title).replace("'", "\\'")
        page_item['es_lasttime'] = last_modified
        page_item['es_urltime'] = last_modified
        page_item['es_loadtime'] = current_date_time
        page_item['es_urlimage'] = image_urls
        page_item['es_abstract'] = json.dumps(infobox_dict, ensure_ascii=False).replace("'", "\\'")
        page_item['es_urltopic'] = json.dumps(toc, ensure_ascii=False).replace("'", "\\'")
        page_item['es_urlcontent'] = json.dumps(contents, ensure_ascii=False).replace("'", "\\'")
        page_item['es_sid'] = get_str_md5(title)
        page_item['es_urlname'] = page_url
        page_item['es_carriertype'] = 'wiki'
        page_item['es_catalog'] = target_type
        yield page_item

    def close(self, reason):
        if self.mysql_db:
            self.mysql_db.close()
