import requests
import os

# 构造请求头
headers = {
    "Accept-Encoding": "identity",
    'session': 'JSESSIONID',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36'
}
urls = []

script_path = r"E:\yuxin\cmip6\script"

eof_str = 'EOF--dataset.file.url.chksum_type.chksum'

if __name__ == "__main__":
    for f_name in os.listdir(script_path):
        if f_name.endswith(".sh"):
            sh_file = open(script_path + '\\' + f_name, "r", encoding='utf-8')
            sh_content = sh_file.read()
            sh_file.close()
            start_index = sh_content.find(eof_str) + len(eof_str)
            end_index = sh_content.rfind(eof_str)
            link_content_lines = sh_content[start_index:end_index].strip().split("\n")
            for line in link_content_lines:
                link = line.split()[1][1:-1]
                download_file_name = link.split("/")[-1]
                r = requests.get(link, stream=True, headers=headers)
                file_size_str = r.headers['Content-Length']  # 提取出来的是个数字str
                print("{}: {}".format(download_file_name, file_size_str))
