Compare commits
No commits in common. "910794aff730d92669b4ce56f908f7a78d69a05d" and "9a36e9c5b5f0cb002908abf53ad332ef8833319b" have entirely different histories.
910794aff7
...
9a36e9c5b5
171
research/pdf_downloader/decode-url-for-rodong-news.py
Normal file
171
research/pdf_downloader/decode-url-for-rodong-news.py
Normal file
@ -0,0 +1,171 @@
|
||||
import mysql.connector
|
||||
import base64
|
||||
import urllib.parse
|
||||
import re
|
||||
|
||||
# === 数据库配置 ===
|
||||
DB_CONFIG = {
|
||||
'host': '47.113.231.200',
|
||||
'port': 28089,
|
||||
'user': 'root',
|
||||
'password': 'passok123A',
|
||||
'database': 'dsp',
|
||||
'charset': 'utf8mb4',
|
||||
}
|
||||
|
||||
|
||||
def decode_rodong_url(url):
|
||||
"""
|
||||
从朝鲜劳动新闻URL中提取并Base64解码参数部分
|
||||
示例输入: http://www.rodong.rep.kp/cn/index.php?MTJAMjAyNi0wMS0wNS0wMDJAMUAxQEAwQDNA==
|
||||
输出: '12@2026-01-05-002@1@1@@0@37@' 或 None（若无法解析）
|
||||
"""
|
||||
if not url or 'index.php?' not in url:
|
||||
return None
|
||||
|
||||
try:
|
||||
# 方法1：使用 urllib.parse 解析
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
query = parsed.query
|
||||
|
||||
# 如果 query 为空，尝试用正则兜底（应对非常规URL）
|
||||
if not query:
|
||||
match = re.search(r'index\.php\?([A-Za-z0-9+/=]+)', url)
|
||||
if match:
|
||||
query = match.group(1)
|
||||
else:
|
||||
return None
|
||||
|
||||
# Base64 解码
|
||||
decoded_bytes = base64.b64decode(query)
|
||||
decoded_str = decoded_bytes.decode('utf-8')
|
||||
return decoded_str
|
||||
|
||||
except Exception as e:
|
||||
# 记录错误但不中断整体流程
|
||||
print(f" 解码失败 (URL: {url[:60]}...): {e}")
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
# 连接数据库
|
||||
conn = mysql.connector.connect(**DB_CONFIG)
|
||||
cursor = conn.cursor(buffered=True)
|
||||
|
||||
# 查询所有需要处理的记录（只处理包含 index.php? 的 URL）
|
||||
print("正在查询待处理的新闻记录...")
|
||||
cursor.execute("""
|
||||
SELECT es_sid, es_urlname
|
||||
FROM indeximos
|
||||
WHERE es_sitename = '劳动新闻'
|
||||
AND (es_tags IS NULL OR es_tags = '')
|
||||
""")
|
||||
records = cursor.fetchall()
|
||||
|
||||
if not records:
|
||||
print("没有找到需要处理的记录。")
|
||||
return
|
||||
|
||||
print(f"共找到 {len(records)} 条待处理记录。")
|
||||
|
||||
updated_count = 0
|
||||
for i, (es_sid, es_urlname) in enumerate(records, 1):
|
||||
print(f"[{i}/{len(records)}] 处理 ID={es_sid} ...", end=" ")
|
||||
|
||||
decoded = decode_rodong_url(es_urlname)
|
||||
if decoded is not None:
|
||||
# 更新 es_tags 字段
|
||||
update_query = "UPDATE indeximos SET es_tags = %s WHERE es_sid = %s"
|
||||
cursor.execute(update_query, (decoded, es_sid))
|
||||
conn.commit()
|
||||
updated_count += 1
|
||||
print(f"成功 → {decoded[:50]}{'...' if len(decoded) > 50 else ''}")
|
||||
else:
|
||||
print("跳过（无法解码）")
|
||||
|
||||
print(f"\n✅ 完成！共更新 {updated_count} 条记录。")
|
||||
|
||||
except mysql.connector.Error as db_err:
|
||||
print(f"❌ 数据库错误: {db_err}")
|
||||
except Exception as e:
|
||||
print(f"❌ 脚本执行出错: {e}")
|
||||
finally:
|
||||
if 'cursor' in locals():
|
||||
cursor.close()
|
||||
if 'conn' in locals() and conn.is_connected():
|
||||
conn.close()
|
||||
print("数据库连接已关闭。")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
# 动态替换 SQL 中的表名（注意：表名不能用参数化，需手动拼接，但确保安全）
|
||||
# 为安全起见，可加校验
|
||||
if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', 'indeximos'):
|
||||
raise ValueError("表名包含非法字符！")
|
||||
|
||||
# 临时替换函数中的表名（更优雅的方式是传参，此处为简洁）
|
||||
import sys
|
||||
|
||||
module = sys.modules[__name__]
|
||||
|
||||
|
||||
# 修改 main 函数中的 SQL（通过字符串替换）
|
||||
# 实际建议：将表名作为全局变量或参数传递
|
||||
|
||||
# 更简单做法：在 main() 上方定义 TABLE_NAME，然后在 SQL 中直接引用
|
||||
# 我们重写 main 函数内部逻辑以支持变量表名
|
||||
|
||||
# 重新定义带表名参数的主逻辑
|
||||
def main_with_table(table_name):
|
||||
try:
|
||||
conn = mysql.connector.connect(**DB_CONFIG)
|
||||
cursor = conn.cursor(buffered=True)
|
||||
|
||||
# 查询
|
||||
query_sql = f"""
|
||||
SELECT es_sid, es_urlname
|
||||
FROM `{table_name}`
|
||||
WHERE es_urlname LIKE '%index.php?%'
|
||||
AND (es_tags IS NULL OR es_tags = '')
|
||||
"""
|
||||
cursor.execute(query_sql)
|
||||
records = cursor.fetchall()
|
||||
|
||||
if not records:
|
||||
print("没有找到需要处理的记录。")
|
||||
return
|
||||
|
||||
print(f"共找到 {len(records)} 条待处理记录。")
|
||||
|
||||
updated_count = 0
|
||||
for i, (es_sid, es_urlname) in enumerate(records, 1):
|
||||
print(f"[{i}/{len(records)}] 处理 ID={es_sid} ...", end=" ")
|
||||
|
||||
decoded = decode_rodong_url(es_urlname)
|
||||
if decoded is not None:
|
||||
update_sql = f"UPDATE `{table_name}` SET es_tags = %s WHERE es_sid = %s"
|
||||
cursor.execute(update_sql, (decoded, es_sid))
|
||||
conn.commit()
|
||||
updated_count += 1
|
||||
print(f"成功 → {decoded[:50]}{'...' if len(decoded) > 50 else ''}")
|
||||
else:
|
||||
print("跳过（无法解码）")
|
||||
|
||||
print(f"\n✅ 完成！共更新 {updated_count} 条记录。")
|
||||
|
||||
except mysql.connector.Error as db_err:
|
||||
print(f"❌ 数据库错误: {db_err}")
|
||||
except Exception as e:
|
||||
print(f"❌ 脚本执行出错: {e}")
|
||||
finally:
|
||||
if 'cursor' in locals():
|
||||
cursor.close()
|
||||
if 'conn' in locals() and conn.is_connected():
|
||||
conn.close()
|
||||
print("数据库连接已关闭。")
|
||||
|
||||
|
||||
# 执行
|
||||
main_with_table('indeximos')
|
||||
@ -75,8 +75,7 @@ def main():
|
||||
FROM indeximos
|
||||
WHERE es_srcname IN ({placeholders})
|
||||
AND es_urlname IS NOT NULL
|
||||
AND es_urlname != ''
|
||||
AND es_loadtime > '2026-01-16 10:40:00'
|
||||
AND es_urlname != ''
|
||||
"""
|
||||
cursor.execute(query, TARGET_SRCNAMES)
|
||||
records = cursor.fetchall()
|
||||
|
||||
@ -63,7 +63,6 @@ def get_chinese_records(cursor) -> List[Tuple]:
|
||||
WHERE es_srcname IN ({placeholders})
|
||||
AND es_urltitle IS NOT NULL AND TRIM(es_urltitle) != ''
|
||||
AND es_urltime IS NOT NULL
|
||||
AND es_loadtime > '2026-01-16 10:40:00'
|
||||
"""
|
||||
cursor.execute(query, TARGET_SRCNAMES)
|
||||
return cursor.fetchall()
|
||||
@ -74,12 +73,11 @@ def get_foreign_candidates_by_time(cursor, pub_time) -> List[Tuple]:
|
||||
获取同一发布时间的所有外文候选记录（要求 es_abstract 不为空）
|
||||
"""
|
||||
query = """
|
||||
SELECT es_sid, es_title, es_urltitle, es_urlcontent
|
||||
SELECT es_sid, es_abstract, es_urltitle, es_urlcontent
|
||||
FROM indeximos
|
||||
WHERE es_urltime = %s
|
||||
AND es_title IS NOT NULL AND TRIM(es_title) != ''
|
||||
AND es_abstract IS NOT NULL AND TRIM(es_abstract) != ''
|
||||
AND es_urlcontent IS NOT NULL
|
||||
AND es_loadtime > '2026-01-16 10:40:00'
|
||||
"""
|
||||
cursor.execute(query, (pub_time,))
|
||||
return cursor.fetchall()
|
||||
|
||||
@ -1,175 +0,0 @@
|
||||
import time
|
||||
from typing import List, Tuple, Optional
|
||||
|
||||
import pymysql
|
||||
import requests
|
||||
|
||||
# ================== 配置区 ==================
|
||||
|
||||
# 数据库配置
|
||||
DB_CONFIG = {
|
||||
'host': '47.113.231.200',
|
||||
'port': 28089,
|
||||
'user': 'root',
|
||||
'password': 'passok123A',
|
||||
'database': 'dsp',
|
||||
'charset': 'utf8mb4',
|
||||
'autocommit': False # 手动控制事务
|
||||
}
|
||||
|
||||
# 翻译 API 地址（替换为你的服务器 IP 或域名）
|
||||
TRANSLATE_API_URL = "http://47.113.231.200:28081/translate"
|
||||
|
||||
# 指定时间（格式：YYYY-MM-DD HH:MM:SS）
|
||||
LOADTIME_AFTER = "2026-01-16 10:40:00"
|
||||
|
||||
# 目标站点列表
|
||||
TARGET_SRCNAMES = [
|
||||
'http://www.rodong.rep.kp/ko/index.php?MUBAMUAxQA==',
|
||||
'http://www.kcna.kp/kp/category/articles/q/5394b80bdae203fadef02522cfb578c0.kcmsf',
|
||||
'https://energynow.com/category/press_releases/',
|
||||
'https://www.fao.org/newsroom/en' # 添加你的站点
|
||||
]
|
||||
|
||||
# 单次请求间隔（秒），避免 API 被限流
|
||||
REQUEST_DELAY = 1
|
||||
|
||||
# 最大文本长度（与 API 一致）
|
||||
MAX_TEXT_LENGTH = 5000
|
||||
|
||||
|
||||
def normalize_newlines(text: str) -> str:
|
||||
"""将 \r\n 和 \r 统一转换为 \n"""
|
||||
if not text:
|
||||
return text
|
||||
return text.replace('\r\n', '\n').replace('\r', '\n')
|
||||
|
||||
|
||||
def translate_single(text: str, source_lang: str = "auto", target_lang: str = "zh") -> Optional[str]:
|
||||
"""翻译单段文本，失败返回 None"""
|
||||
if not text or not text.strip():
|
||||
return ""
|
||||
|
||||
payload = {
|
||||
"text": text[:MAX_TEXT_LENGTH],
|
||||
"source_lang": source_lang,
|
||||
"target_lang": target_lang
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.post(TRANSLATE_API_URL, json=payload, timeout=10)
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
return result.get("translated_text")
|
||||
except Exception as e:
|
||||
print(f"⚠️ 翻译失败: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def translate_content_with_paragraphs(content: str) -> str:
|
||||
"""
|
||||
按段落翻译内容，支持容错：
|
||||
- 某段失败 → 跳过该段（保留空行或原文）
|
||||
- 返回拼接后的完整内容
|
||||
"""
|
||||
if not content:
|
||||
return ""
|
||||
|
||||
# 标准化换行符
|
||||
content = normalize_newlines(content)
|
||||
paragraphs = content.split('\n')
|
||||
translated_paragraphs = []
|
||||
|
||||
for para in paragraphs:
|
||||
if not para.strip():
|
||||
# 保留空行
|
||||
translated_paragraphs.append("")
|
||||
continue
|
||||
|
||||
trans = translate_single(para)
|
||||
if trans is None:
|
||||
# 段落翻译失败：跳过该段（可选：保留原文或留空）
|
||||
print(f" ⚠️ 段落翻译失败，跳过: {para[:30]}...")
|
||||
translated_paragraphs.append("") # 或 append(para) 保留原文
|
||||
else:
|
||||
translated_paragraphs.append(trans)
|
||||
|
||||
time.sleep(REQUEST_DELAY)
|
||||
|
||||
return '\n'.join(translated_paragraphs)
|
||||
|
||||
|
||||
# ================== 数据库操作 ==================
|
||||
|
||||
def update_record(cursor, es_sid: int, new_title: str, new_content: str):
|
||||
update_query = """
|
||||
UPDATE indeximos
|
||||
SET es_title = % s, es_content = % s
|
||||
WHERE es_sid = % s
|
||||
"""
|
||||
cursor.execute(update_query, (new_title, new_content, es_sid))
|
||||
|
||||
|
||||
# ================== 主逻辑 ==================
|
||||
|
||||
def main():
|
||||
conn = pymysql.connect(**DB_CONFIG)
|
||||
cursor = conn.cursor()
|
||||
|
||||
try:
|
||||
placeholders = ','.join(['%s'] * len(TARGET_SRCNAMES))
|
||||
query = f"""
|
||||
SELECT es_sid, es_urltitle, es_urlcontent
|
||||
FROM indeximos
|
||||
WHERE es_loadtime > %s
|
||||
AND (es_title IS NULL OR TRIM(es_title) = '')
|
||||
AND es_srcname IN ({placeholders})
|
||||
AND LENGTH(es_video) > 5
|
||||
"""
|
||||
params = [LOADTIME_AFTER] + TARGET_SRCNAMES
|
||||
cursor.execute(query, params)
|
||||
records: List[Tuple] = cursor.fetchall()
|
||||
|
||||
total = len(records)
|
||||
print(f"✅ 共找到 {total} 条待翻译记录")
|
||||
|
||||
if total == 0:
|
||||
return
|
||||
|
||||
success_count = 0
|
||||
|
||||
for idx, (es_sid, urltitle, urlcontent) in enumerate(records, 1):
|
||||
print(f"\n[{idx}/{total}] 处理 es_sid={es_sid}")
|
||||
start_time = time.time()
|
||||
|
||||
# 翻译标题
|
||||
title_trans = translate_single(urltitle) if urltitle else ""
|
||||
if title_trans is None:
|
||||
print(" → 标题翻译失败，跳过整条")
|
||||
continue
|
||||
|
||||
# 翻译内容（按段落，容错）
|
||||
content_trans = translate_content_with_paragraphs(urlcontent)
|
||||
|
||||
# 更新数据库
|
||||
update_record(cursor, es_sid, title_trans, content_trans)
|
||||
success_count += 1
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
print(f" ✅ 翻译成功 | 耗时: {elapsed:.2f}s | 标题: {title_trans[:30]}...")
|
||||
|
||||
conn.commit()
|
||||
|
||||
print(f"\n🎉 完成！成功翻译 {success_count} / {total} 条记录")
|
||||
|
||||
except Exception as e:
|
||||
conn.rollback()
|
||||
print(f"❌ 发生错误: {e}")
|
||||
raise
|
||||
finally:
|
||||
cursor.close()
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
x
Reference in New Issue
Block a user