import json
import os

response_root_path = r'E:\yuxin\cmip20240424'
var_ids = ['tas']
skip_path = []

if __name__ == "__main__":
    for var_id in var_ids:
        var_path = os.path.join(response_root_path, var_id)
        source_ids = os.listdir(var_path)
        for source_id in source_ids:
            # if source_id in skip_path:
            #     continue
            source_path = os.path.join(var_path, source_id)
            rsp_file_names = os.listdir(source_path)
            for rsp_file_name in rsp_file_names:
                rsp_file = open(os.path.join(source_path, rsp_file_name), 'r', encoding='utf-8')
                rsp_body = json.loads(rsp_file.read())
                rsp_file.close()
                docs = rsp_body['response']['docs']
                csv_file = open(os.path.join(response_root_path, 'dl_url.csv'), 'a', encoding='utf-8')
                for doc in docs:
                    try:
                        csv_file.write(
                            f"{doc['data_node']},{doc['variable_id'][0]},{doc['source_id'][0]},{doc['size']},{doc['instance_id']},{doc['url'][0]}\n")
                    except Exception as e:
                        print(repr(e))
                csv_file.close()