import os
import time

import pymysql
from lxml import etree
import json
from WikipediaSpiders.utils.parse_utils import ZhParser, JpParser, EnParser
from urllib.parse import unquote

from WikipediaSpiders.utils.string_utils import get_str_md5

page_file_path = r"E:\yuxin\ship-wiki\pages"

if __name__ == '__main__':
    db = pymysql.connect(host='39.101.194.63', port=23306,
                         user='root', passwd='passok123A', db='nfm', charset='utf8mb4')
    cursor = db.cursor()
    count = 0
    for _path in os.listdir(page_file_path):
        if _path.endswith('.html'):
            count += 1
            page_info = {}
            target_file = open(page_file_path + '\\' + _path, mode='r', encoding='utf-8')
            html_content = target_file.read().replace('\n', '')
            target_file.close()
            selector = etree.HTML(html_content)
            title = ''.join(selector.xpath("//h1[@id='firstHeading']//text()"))
            page_url = selector.xpath("//link[@rel='canonical']/@href")[0]
            page_url = unquote(page_url)
            if page_url.startswith('https://zh.wikipedia.org/'):
                page_parser = ZhParser()
            elif page_url.startswith('https://ja.wikipedia.org/'):
                page_parser = JpParser()
            elif page_url.startswith('https://en.wikipedia.org/'):
                page_parser = EnParser()
            else:
                continue
            photo, infobox_dict = page_parser.parse_infobox_info(selector)
            toc = page_parser.parse_toc(selector)
            contents = page_parser.parse_contents(selector)
            last_modified = page_parser.parse_modified_time(selector)
            current_date_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
            downloaded_image_urls = []
            image_urls = []
            for _ in photo:
                downloaded_image_urls.append("https:" + _)
                image_urls.append('./images/' + _.split('/')[-1])
            for content in contents:
                if 'images' in content:
                    content_images = content['images']
                    for image in content_images:
                        downloaded_image_urls.append('https:' + image)
                        image_urls.append('./images/' + image.split('/')[-1])
            db_item = {
                'id': get_str_md5(title),
                'title': title.replace("'", "\\'"),
                'info_table': json.dumps(infobox_dict, ensure_ascii=False).replace("'", "\\'"),
                'photo': json.dumps(image_urls, ensure_ascii=False).replace("'", "\\'"),
                'toc': json.dumps(toc, ensure_ascii=False).replace("'", "\\'"),
                'contents': json.dumps(contents, ensure_ascii=False).replace("'", "\\'"),
                'last_modified': last_modified,
                'wiki_source': page_url.replace("'", "\\'"),
                'crawl_time': current_date_time
            }
            db_cols = []
            db_vals = []
            for _ in db_item:
                db_cols.append(_)
                db_vals.append("'{}'".format(db_item[_]))
            SQL_INSERT = "INSERT INTO wiki.page_info ({}) VALUES ({})".format(", ".join(db_cols), ", ".join(db_vals))
            print("[No. {}] {}".format(count, title))
            for _url in downloaded_image_urls:
                print(_url)
            try:
                cursor.execute(SQL_INSERT)
            except Exception as e:
                print(repr(e))
    db.commit()
    db.close()
