80 lines
3.2 KiB
Python
80 lines
3.2 KiB
Python
|
import json
|
||
|
|
import random
|
||
|
|
import time
|
||
|
|
|
||
|
|
import pymysql
|
||
|
|
|
||
|
|
from api_request import ApiRequest
|
||
|
|
|
||
|
|
|
||
|
|
def exec_sql(local_db, local_cursor, sql):
|
||
|
|
try:
|
||
|
|
local_cursor.execute(sql)
|
||
|
|
local_db.commit()
|
||
|
|
except:
|
||
|
|
local_db = pymysql.connect(host='39.101.194.63', port=23306,
|
||
|
|
user='root', passwd='passok123A', db='jd_data', charset='utf8mb4')
|
||
|
|
local_cursor = local_db.cursor()
|
||
|
|
local_cursor.execute(sql)
|
||
|
|
local_db.commit()
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
db = pymysql.connect(host='39.101.194.63', port=23306,
|
||
|
|
user='root', passwd='passok123A', db='jd_data', charset='utf8mb4')
|
||
|
|
cursor = db.cursor()
|
||
|
|
ar = ApiRequest()
|
||
|
|
|
||
|
|
while True:
|
||
|
|
try:
|
||
|
|
sql_select = "SELECT id, experiment, variable, model, level, data_size, downloaded, downloaded_time, update_time " \
|
||
|
|
"FROM jd_data.data_detail WHERE available = 1 AND downloaded = 0 " \
|
||
|
|
"AND (update_time is NULL or update_time < '2023-04-15')"
|
||
|
|
cursor.execute(sql_select)
|
||
|
|
result = cursor.fetchall()
|
||
|
|
total_num = len(result)
|
||
|
|
if total_num == 0:
|
||
|
|
break
|
||
|
|
else:
|
||
|
|
print("剩余{}个大文件".format(total_num))
|
||
|
|
row = result[random.choice(range(total_num))]
|
||
|
|
row_id = row[0]
|
||
|
|
experiment = row[1]
|
||
|
|
variable = row[2]
|
||
|
|
model = row[3]
|
||
|
|
level = row[4]
|
||
|
|
data_size = row[5]
|
||
|
|
print('%s, %s, %s, %s' % (row_id, experiment, variable, model))
|
||
|
|
param = {
|
||
|
|
'experiment': experiment,
|
||
|
|
'variable': variable,
|
||
|
|
'model': model,
|
||
|
|
'level': level
|
||
|
|
}
|
||
|
|
reply = ar.get_reply(param)
|
||
|
|
if reply:
|
||
|
|
remarks = json.dumps(reply)
|
||
|
|
location = ''
|
||
|
|
content_length = 0
|
||
|
|
if reply['state'] == 'completed':
|
||
|
|
location = reply['location']
|
||
|
|
content_length = reply['content_length']
|
||
|
|
sent_to_rmq_at = 'null'
|
||
|
|
if 'sent_to_rmq_at' in reply:
|
||
|
|
sent_to_rmq_at = "'" + reply['sent_to_rmq_at'].replace("T", " ").replace("Z", "") + "'"
|
||
|
|
else:
|
||
|
|
sent_to_rmq_at = "'" + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + "'"
|
||
|
|
print(reply)
|
||
|
|
sql_update_status = "UPDATE jd_data.data_detail " \
|
||
|
|
"SET available = 1, url = '%s', data_size = %d, update_time = %s, remarks = '%s' " \
|
||
|
|
"WHERE id = %d" \
|
||
|
|
% (
|
||
|
|
location, content_length, sent_to_rmq_at, remarks.replace("'", "\\'"),
|
||
|
|
row_id)
|
||
|
|
exec_sql(db, cursor, sql_update_status)
|
||
|
|
except:
|
||
|
|
db = pymysql.connect(host='39.101.194.63', port=23306,
|
||
|
|
user='root', passwd='passok123A', db='jd_data', charset='utf8mb4')
|
||
|
|
cursor = db.cursor()
|
||
|
|
db.close()
|