import os
import wget
from urllib.parse import unquote

download_path = '/root/wiki/pages'
urls_file_path = "/root/wiki/wiki-links.txt"


def valid_file_name(filename):
    for _ in ['"', '/', '\\', '|', ':', '?', '<', '>', '*']:
        filename = filename.replace(_, "-")
    return filename


if __name__ == "__main__":
    urls_file = open(urls_file_path, 'r', encoding='utf-8')
    urls = urls_file.read().split('\n')
    urls_file.close()
    count = 0
    for url in urls:
        url = unquote(url)
        file_name = url.split('/')[-1]
        file_name = valid_file_name(file_name)
        file_name = file_name + '.html'
        file_path = download_path + '/' + file_name
        if not os.path.exists(file_path):
            try:
                count += 1
                wget.download(url, out=file_path)
                print(f"[No {count}] Downloaded {file_name}")
            except Exception as e:
                print(url + ' ERROR: ' + repr(e))
