修复部分字段问题

dsp/src/main/java/com/jsc/dsp/utils/AutoExportAndUpload.java

@@ -81,7 +81,6 @@ public class AutoExportAndUpload {
         String currentLoadTime = StringUtils.DateToString(new Date());
         String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd"));
         databaseConnector.twitterToXlsx(startTime);
-        copyPagesFiles(startTime, currentLoadTime);
         configService.setConfigValueByName("twitter_last_loadtime", currentLoadTime);
         String zipFileName = "data_twitter-" + timestamp + "-001.zip";
         String zipFileFullName = backupFilePath + File.separator + zipFileName;

dsp/src/main/java/com/jsc/dsp/utils/DatabaseConnector.java

@@ -26,7 +26,9 @@ import java.nio.file.Paths;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.List;
+
 import com.fasterxml.jackson.databind.JsonNode;
+
 import java.util.ArrayList;
 
 @Service
@@ -133,8 +135,16 @@ public class DatabaseConnector {
                         row.createCell(9).setCellValue(item.getEsLoadtime());
                         row.createCell(10).setCellValue(item.getEsSitename());
                         row.createCell(11).setCellValue(item.getEsSrcname());
-                        row.createCell(12).setCellValue(item.getEsUrlcontent());
-                        row.createCell(13).setCellValue(item.getEsUrlcontentTranslate());
+                        if (item.getEsUrlcontent().length() > 30000) {
+                            row.createCell(12).setCellValue(item.getEsUrlcontent().substring(0, 30000));
+                        } else {
+                            row.createCell(12).setCellValue(item.getEsUrlcontent());
+                        }
+                        if (item.getEsUrlcontentTranslate().length() > 30000) {
+                            row.createCell(13).setCellValue(item.getEsUrlcontentTranslate().substring(0, 30000));
+                        } else {
+                            row.createCell(13).setCellValue(item.getEsUrlcontentTranslate());
+                        }
                         row.createCell(14).setCellValue(item.getEsUrlimage());
                         row.createCell(15).setCellValue(item.getEsUrlname());
                         row.createCell(16).setCellValue(item.getEsUrltime());
@@ -314,7 +324,6 @@ public class DatabaseConnector {
     }
 
 
-
     public String extractFilenamesFromJsonArray(String jsonStr) {
         if (jsonStr == null || jsonStr.trim().isEmpty()) {
             return "";

dsp/src/main/java/com/jsc/dsp/utils/TodistParseUtil.java

@@ -0,0 +1,94 @@
+package com.jsc.dsp.utils;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.google.protobuf.Descriptors;
+import com.google.protobuf.GeneratedMessageV3;
+import com.google.protobuf.InvalidProtocolBufferException;
+import com.jsc.dsp.proto.EsOuterClass;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Map;
+
+/**
+ * 备选方案：使用 FastJSON 手动转换（无额外依赖）
+ */
+public class TodistParseUtil {
+
+    public static String protobufToJson(EsOuterClass.EsSets esSets) {
+        JSONObject root = new JSONObject();
+
+        // 处理 repeated Es 字段
+        JSONArray esArray = new JSONArray();
+        for (EsOuterClass.Es es : esSets.getEsList()) {
+            esArray.add(messageToJson(es));
+        }
+        root.put("es", esArray);
+
+        return JSON.toJSONString(root, true); // pretty format
+    }
+
+    private static JSONObject messageToJson(GeneratedMessageV3 message) {
+        JSONObject json = new JSONObject();
+        Map<Descriptors.FieldDescriptor, Object> fields = message.getAllFields();
+
+        for (Map.Entry<Descriptors.FieldDescriptor, Object> entry : fields.entrySet()) {
+            Descriptors.FieldDescriptor field = entry.getKey();
+            Object value = entry.getValue();
+
+            if (field.isRepeated()) {
+                JSONArray array = new JSONArray();
+                if (value instanceof Iterable) {
+                    for (Object item : (Iterable<?>) value) {
+                        array.add(convertFieldValue(item));
+                    }
+                }
+                json.put(field.getName(), array);
+            } else {
+                json.put(field.getName(), convertFieldValue(value));
+            }
+        }
+        return json;
+    }
+
+    private static Object convertFieldValue(Object value) {
+        if (value instanceof GeneratedMessageV3) {
+            return messageToJson((GeneratedMessageV3) value);
+        }
+        // 其他类型直接返回（Protobuf 基本类型可被 FastJSON 识别）
+        return value;
+    }
+
+    public static void main(String[] args) {
+
+        String filePath = "C:/Users/yuxin/Documents/xwechat_files/wxid_dtvj9sibla0d21_9cb3/msg/file/2026-02/public_info_data_1770264282958.todist";
+        try {
+            // 1. 流式读取文件（避免大文件 OOM）
+            byte[] data = Files.readAllBytes(Paths.get(filePath));
+
+            // 2. Protobuf 反序列化
+            EsOuterClass.EsSets esSets = EsOuterClass.EsSets.parseFrom(data);
+            System.out.println("✅ 成功解析 EsSets，共 " + esSets.getEsCount() + " 条记录");
+
+            // 3. 转换为 JSON（使用 Protobuf 原生 JsonFormat）
+            String json = protobufToJson(esSets);
+
+            // 4. 输出格式化 JSON
+            System.out.println("/n📄 JSON Output:");
+            System.out.println(json);
+
+        } catch (InvalidProtocolBufferException e) {
+            System.err.println("❌ Protobuf 解析失败: " + e.getMessage());
+            e.printStackTrace();
+        } catch (IOException e) {
+            System.err.println("❌ 文件读取失败: " + e.getMessage());
+            e.printStackTrace();
+        } catch (Exception e) {
+            System.err.println("❌ 未知错误: " + e.getMessage());
+            e.printStackTrace();
+        }
+    }
+}

dsp/src/main/resources/application.yml

@@ -63,7 +63,7 @@ switch:
   enable-storage-service: false
   enable-file-dl-service: false
   enable-protobuf-service: false
-  auto-export-and-upload: true
+  auto-export-and-upload: false
 
 ftp:
   host: 144.34.185.108