Create translate-news.py

执行翻译

research/pdf_downloader/decode-url-for-rodong-news.py

@@ -1,171 +0,0 @@
-import mysql.connector
-import base64
-import urllib.parse
-import re
-
-# === 数据库配置 ===
-DB_CONFIG = {
-    'host': '47.113.231.200',
-    'port': 28089,
-    'user': 'root',
-    'password': 'passok123A',
-    'database': 'dsp',
-    'charset': 'utf8mb4',
-}
-
-
-def decode_rodong_url(url):
-    """
-    从朝鲜劳动新闻URL中提取并Base64解码参数部分
-    示例输入: http://www.rodong.rep.kp/cn/index.php?MTJAMjAyNi0wMS0wNS0wMDJAMUAxQEAwQDNA==
-    输出: '12@2026-01-05-002@1@1@@0@37@' 或 None（若无法解析）
-    """
-    if not url or 'index.php?' not in url:
-        return None
-
-    try:
-        # 方法1：使用 urllib.parse 解析
-        parsed = urllib.parse.urlparse(url)
-        query = parsed.query
-
-        # 如果 query 为空，尝试用正则兜底（应对非常规URL）
-        if not query:
-            match = re.search(r'index\.php\?([A-Za-z0-9+/=]+)', url)
-            if match:
-                query = match.group(1)
-            else:
-                return None
-
-        # Base64 解码
-        decoded_bytes = base64.b64decode(query)
-        decoded_str = decoded_bytes.decode('utf-8')
-        return decoded_str
-
-    except Exception as e:
-        # 记录错误但不中断整体流程
-        print(f"  解码失败 (URL: {url[:60]}...): {e}")
-        return None
-
-
-def main():
-    try:
-        # 连接数据库
-        conn = mysql.connector.connect(**DB_CONFIG)
-        cursor = conn.cursor(buffered=True)
-
-        # 查询所有需要处理的记录（只处理包含 index.php? 的 URL）
-        print("正在查询待处理的新闻记录...")
-        cursor.execute("""
-            SELECT es_sid, es_urlname 
-            FROM indeximos 
-            WHERE es_sitename = '劳动新闻' 
-              AND (es_tags IS NULL OR es_tags = '')
-        """)
-        records = cursor.fetchall()
-
-        if not records:
-            print("没有找到需要处理的记录。")
-            return
-
-        print(f"共找到 {len(records)} 条待处理记录。")
-
-        updated_count = 0
-        for i, (es_sid, es_urlname) in enumerate(records, 1):
-            print(f"[{i}/{len(records)}] 处理 ID={es_sid} ...", end=" ")
-
-            decoded = decode_rodong_url(es_urlname)
-            if decoded is not None:
-                # 更新 es_tags 字段
-                update_query = "UPDATE indeximos SET es_tags = %s WHERE es_sid = %s"
-                cursor.execute(update_query, (decoded, es_sid))
-                conn.commit()
-                updated_count += 1
-                print(f"成功 → {decoded[:50]}{'...' if len(decoded) > 50 else ''}")
-            else:
-                print("跳过（无法解码）")
-
-        print(f"\n✅ 完成！共更新 {updated_count} 条记录。")
-
-    except mysql.connector.Error as db_err:
-        print(f"❌ 数据库错误: {db_err}")
-    except Exception as e:
-        print(f"❌ 脚本执行出错: {e}")
-    finally:
-        if 'cursor' in locals():
-            cursor.close()
-        if 'conn' in locals() and conn.is_connected():
-            conn.close()
-            print("数据库连接已关闭。")
-
-
-if __name__ == "__main__":
-
-    # 动态替换 SQL 中的表名（注意：表名不能用参数化，需手动拼接，但确保安全）
-    # 为安全起见，可加校验
-    if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', 'indeximos'):
-        raise ValueError("表名包含非法字符！")
-
-    # 临时替换函数中的表名（更优雅的方式是传参，此处为简洁）
-    import sys
-
-    module = sys.modules[__name__]
-
-
-    # 修改 main 函数中的 SQL（通过字符串替换）
-    # 实际建议：将表名作为全局变量或参数传递
-
-    # 更简单做法：在 main() 上方定义 TABLE_NAME，然后在 SQL 中直接引用
-    # 我们重写 main 函数内部逻辑以支持变量表名
-
-    # 重新定义带表名参数的主逻辑
-    def main_with_table(table_name):
-        try:
-            conn = mysql.connector.connect(**DB_CONFIG)
-            cursor = conn.cursor(buffered=True)
-
-            # 查询
-            query_sql = f"""
-                SELECT es_sid, es_urlname 
-                FROM `{table_name}` 
-                WHERE es_urlname LIKE '%index.php?%'
-                  AND (es_tags IS NULL OR es_tags = '')
-            """
-            cursor.execute(query_sql)
-            records = cursor.fetchall()
-
-            if not records:
-                print("没有找到需要处理的记录。")
-                return
-
-            print(f"共找到 {len(records)} 条待处理记录。")
-
-            updated_count = 0
-            for i, (es_sid, es_urlname) in enumerate(records, 1):
-                print(f"[{i}/{len(records)}] 处理 ID={es_sid} ...", end=" ")
-
-                decoded = decode_rodong_url(es_urlname)
-                if decoded is not None:
-                    update_sql = f"UPDATE `{table_name}` SET es_tags = %s WHERE es_sid = %s"
-                    cursor.execute(update_sql, (decoded, es_sid))
-                    conn.commit()
-                    updated_count += 1
-                    print(f"成功 → {decoded[:50]}{'...' if len(decoded) > 50 else ''}")
-                else:
-                    print("跳过（无法解码）")
-
-            print(f"\n✅ 完成！共更新 {updated_count} 条记录。")
-
-        except mysql.connector.Error as db_err:
-            print(f"❌ 数据库错误: {db_err}")
-        except Exception as e:
-            print(f"❌ 脚本执行出错: {e}")
-        finally:
-            if 'cursor' in locals():
-                cursor.close()
-            if 'conn' in locals() and conn.is_connected():
-                conn.close()
-                print("数据库连接已关闭。")
-
-
-    # 执行
-    main_with_table('indeximos')

research/pdf_downloader/set_raw_title_kcna.py

@@ -75,7 +75,8 @@ def main():
             FROM indeximos 
             WHERE es_srcname IN ({placeholders})
               AND es_urlname IS NOT NULL 
-              AND es_urlname != ''
+              AND es_urlname != '' 
+              AND es_loadtime > '2026-01-16 10:40:00'
         """
         cursor.execute(query, TARGET_SRCNAMES)
         records = cursor.fetchall()

research/pdf_downloader/set_raw_title_rodong.py

@@ -63,6 +63,7 @@ def get_chinese_records(cursor) -> List[Tuple]:
         WHERE es_srcname IN ({placeholders})
           AND es_urltitle IS NOT NULL AND TRIM(es_urltitle) != ''
           AND es_urltime IS NOT NULL
+          AND es_loadtime > '2026-01-16 10:40:00'
     """
     cursor.execute(query, TARGET_SRCNAMES)
     return cursor.fetchall()
@@ -73,11 +74,12 @@ def get_foreign_candidates_by_time(cursor, pub_time) -> List[Tuple]:
     获取同一发布时间的所有外文候选记录（要求 es_abstract 不为空）
     """
     query = """
-        SELECT es_sid, es_abstract, es_urltitle, es_urlcontent
+        SELECT es_sid, es_title, es_urltitle, es_urlcontent
         FROM indeximos
         WHERE es_urltime = %s
-          AND es_abstract IS NOT NULL AND TRIM(es_abstract) != ''
+          AND es_title IS NOT NULL AND TRIM(es_title) != ''
           AND es_urlcontent IS NOT NULL
+          AND es_loadtime > '2026-01-16 10:40:00'
     """
     cursor.execute(query, (pub_time,))
     return cursor.fetchall()

research/pdf_downloader/translate-news.py

@@ -0,0 +1,175 @@
+import time
+from typing import List, Tuple, Optional
+
+import pymysql
+import requests
+
+# ================== 配置区 ==================
+
+# 数据库配置
+DB_CONFIG = {
+    'host': '47.113.231.200',
+    'port': 28089,
+    'user': 'root',
+    'password': 'passok123A',
+    'database': 'dsp',
+    'charset': 'utf8mb4',
+    'autocommit': False  # 手动控制事务
+}
+
+# 翻译 API 地址（替换为你的服务器 IP 或域名）
+TRANSLATE_API_URL = "http://47.113.231.200:28081/translate"
+
+# 指定时间（格式：YYYY-MM-DD HH:MM:SS）
+LOADTIME_AFTER = "2026-01-16 10:40:00"
+
+# 目标站点列表
+TARGET_SRCNAMES = [
+    'http://www.rodong.rep.kp/ko/index.php?MUBAMUAxQA==',
+    'http://www.kcna.kp/kp/category/articles/q/5394b80bdae203fadef02522cfb578c0.kcmsf',
+    'https://energynow.com/category/press_releases/',
+    'https://www.fao.org/newsroom/en'  # 添加你的站点
+]
+
+# 单次请求间隔（秒），避免 API 被限流
+REQUEST_DELAY = 1
+
+# 最大文本长度（与 API 一致）
+MAX_TEXT_LENGTH = 5000
+
+
+def normalize_newlines(text: str) -> str:
+    """将 \r\n 和 \r 统一转换为 \n"""
+    if not text:
+        return text
+    return text.replace('\r\n', '\n').replace('\r', '\n')
+
+
+def translate_single(text: str, source_lang: str = "auto", target_lang: str = "zh") -> Optional[str]:
+    """翻译单段文本，失败返回 None"""
+    if not text or not text.strip():
+        return ""
+
+    payload = {
+        "text": text[:MAX_TEXT_LENGTH],
+        "source_lang": source_lang,
+        "target_lang": target_lang
+    }
+
+    try:
+        response = requests.post(TRANSLATE_API_URL, json=payload, timeout=10)
+        response.raise_for_status()
+        result = response.json()
+        return result.get("translated_text")
+    except Exception as e:
+        print(f"⚠️ 翻译失败: {e}")
+        return None
+
+
+def translate_content_with_paragraphs(content: str) -> str:
+    """
+    按段落翻译内容，支持容错：
+    - 某段失败 → 跳过该段（保留空行或原文）
+    - 返回拼接后的完整内容
+    """
+    if not content:
+        return ""
+
+    # 标准化换行符
+    content = normalize_newlines(content)
+    paragraphs = content.split('\n')
+    translated_paragraphs = []
+
+    for para in paragraphs:
+        if not para.strip():
+            # 保留空行
+            translated_paragraphs.append("")
+            continue
+
+        trans = translate_single(para)
+        if trans is None:
+            # 段落翻译失败：跳过该段（可选：保留原文或留空）
+            print(f"  ⚠️ 段落翻译失败，跳过: {para[:30]}...")
+            translated_paragraphs.append("")  # 或 append(para) 保留原文
+        else:
+            translated_paragraphs.append(trans)
+
+        time.sleep(REQUEST_DELAY)
+
+    return '\n'.join(translated_paragraphs)
+
+
+# ================== 数据库操作 ==================
+
+def update_record(cursor, es_sid: int, new_title: str, new_content: str):
+    update_query = """
+        UPDATE indeximos 
+        SET es_title = % s, es_content = % s 
+        WHERE es_sid = % s
+    """
+    cursor.execute(update_query, (new_title, new_content, es_sid))
+
+
+# ================== 主逻辑 ==================
+
+def main():
+    conn = pymysql.connect(**DB_CONFIG)
+    cursor = conn.cursor()
+
+    try:
+        placeholders = ','.join(['%s'] * len(TARGET_SRCNAMES))
+        query = f"""
+            SELECT es_sid, es_urltitle, es_urlcontent
+            FROM indeximos
+            WHERE es_loadtime > %s
+              AND (es_title IS NULL OR TRIM(es_title) = '')
+              AND es_srcname IN ({placeholders})
+              AND LENGTH(es_video) > 5
+        """
+        params = [LOADTIME_AFTER] + TARGET_SRCNAMES
+        cursor.execute(query, params)
+        records: List[Tuple] = cursor.fetchall()
+
+        total = len(records)
+        print(f"✅ 共找到 {total} 条待翻译记录")
+
+        if total == 0:
+            return
+
+        success_count = 0
+
+        for idx, (es_sid, urltitle, urlcontent) in enumerate(records, 1):
+            print(f"\n[{idx}/{total}] 处理 es_sid={es_sid}")
+            start_time = time.time()
+
+            # 翻译标题
+            title_trans = translate_single(urltitle) if urltitle else ""
+            if title_trans is None:
+                print("  → 标题翻译失败，跳过整条")
+                continue
+
+            # 翻译内容（按段落，容错）
+            content_trans = translate_content_with_paragraphs(urlcontent)
+
+            # 更新数据库
+            update_record(cursor, es_sid, title_trans, content_trans)
+            success_count += 1
+
+            elapsed = time.time() - start_time
+            print(f"  ✅ 翻译成功 | 耗时: {elapsed:.2f}s | 标题: {title_trans[:30]}...")
+
+            conn.commit()
+
+        print(f"\n🎉 完成！成功翻译 {success_count} / {total} 条记录")
+
+    except Exception as e:
+        conn.rollback()
+        print(f"❌ 发生错误: {e}")
+        raise
+    finally:
+        cursor.close()
+        conn.close()
+
+
+if __name__ == "__main__":
+    main()