From eea15d62e1f56c3cb6ed561589f706c83dea3407 Mon Sep 17 00:00:00 2001 From: yuxin-pc Date: Mon, 9 Feb 2026 11:20:17 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=83=A8=E5=88=86=E5=AD=97?= =?UTF-8?q?=E6=AE=B5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jsc/dsp/utils/AutoExportAndUpload.java | 1 - .../com/jsc/dsp/utils/DatabaseConnector.java | 15 ++- .../com/jsc/dsp/utils/TodistParseUtil.java | 94 +++++++++++++++++++ dsp/src/main/resources/application.yml | 2 +- 4 files changed, 107 insertions(+), 5 deletions(-) create mode 100644 dsp/src/main/java/com/jsc/dsp/utils/TodistParseUtil.java diff --git a/dsp/src/main/java/com/jsc/dsp/utils/AutoExportAndUpload.java b/dsp/src/main/java/com/jsc/dsp/utils/AutoExportAndUpload.java index 763513f..06e7020 100644 --- a/dsp/src/main/java/com/jsc/dsp/utils/AutoExportAndUpload.java +++ b/dsp/src/main/java/com/jsc/dsp/utils/AutoExportAndUpload.java @@ -81,7 +81,6 @@ public class AutoExportAndUpload { String currentLoadTime = StringUtils.DateToString(new Date()); String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd")); databaseConnector.twitterToXlsx(startTime); - copyPagesFiles(startTime, currentLoadTime); configService.setConfigValueByName("twitter_last_loadtime", currentLoadTime); String zipFileName = "data_twitter-" + timestamp + "-001.zip"; String zipFileFullName = backupFilePath + File.separator + zipFileName; diff --git a/dsp/src/main/java/com/jsc/dsp/utils/DatabaseConnector.java b/dsp/src/main/java/com/jsc/dsp/utils/DatabaseConnector.java index f2b7cd5..4d70fe4 100644 --- a/dsp/src/main/java/com/jsc/dsp/utils/DatabaseConnector.java +++ b/dsp/src/main/java/com/jsc/dsp/utils/DatabaseConnector.java @@ -26,7 +26,9 @@ import java.nio.file.Paths; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.List; + import com.fasterxml.jackson.databind.JsonNode; + import java.util.ArrayList; @Service @@ -133,8 +135,16 @@ public class DatabaseConnector { row.createCell(9).setCellValue(item.getEsLoadtime()); row.createCell(10).setCellValue(item.getEsSitename()); row.createCell(11).setCellValue(item.getEsSrcname()); - row.createCell(12).setCellValue(item.getEsUrlcontent()); - row.createCell(13).setCellValue(item.getEsUrlcontentTranslate()); + if (item.getEsUrlcontent().length() > 30000) { + row.createCell(12).setCellValue(item.getEsUrlcontent().substring(0, 30000)); + } else { + row.createCell(12).setCellValue(item.getEsUrlcontent()); + } + if (item.getEsUrlcontentTranslate().length() > 30000) { + row.createCell(13).setCellValue(item.getEsUrlcontentTranslate().substring(0, 30000)); + } else { + row.createCell(13).setCellValue(item.getEsUrlcontentTranslate()); + } row.createCell(14).setCellValue(item.getEsUrlimage()); row.createCell(15).setCellValue(item.getEsUrlname()); row.createCell(16).setCellValue(item.getEsUrltime()); @@ -314,7 +324,6 @@ public class DatabaseConnector { } - public String extractFilenamesFromJsonArray(String jsonStr) { if (jsonStr == null || jsonStr.trim().isEmpty()) { return ""; diff --git a/dsp/src/main/java/com/jsc/dsp/utils/TodistParseUtil.java b/dsp/src/main/java/com/jsc/dsp/utils/TodistParseUtil.java new file mode 100644 index 0000000..d6cca1d --- /dev/null +++ b/dsp/src/main/java/com/jsc/dsp/utils/TodistParseUtil.java @@ -0,0 +1,94 @@ +package com.jsc.dsp.utils; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.google.protobuf.Descriptors; +import com.google.protobuf.GeneratedMessageV3; +import com.google.protobuf.InvalidProtocolBufferException; +import com.jsc.dsp.proto.EsOuterClass; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Map; + +/** + * 备选方案：使用 FastJSON 手动转换（无额外依赖） + */ +public class TodistParseUtil { + + public static String protobufToJson(EsOuterClass.EsSets esSets) { + JSONObject root = new JSONObject(); + + // 处理 repeated Es 字段 + JSONArray esArray = new JSONArray(); + for (EsOuterClass.Es es : esSets.getEsList()) { + esArray.add(messageToJson(es)); + } + root.put("es", esArray); + + return JSON.toJSONString(root, true); // pretty format + } + + private static JSONObject messageToJson(GeneratedMessageV3 message) { + JSONObject json = new JSONObject(); + Map fields = message.getAllFields(); + + for (Map.Entry entry : fields.entrySet()) { + Descriptors.FieldDescriptor field = entry.getKey(); + Object value = entry.getValue(); + + if (field.isRepeated()) { + JSONArray array = new JSONArray(); + if (value instanceof Iterable) { + for (Object item : (Iterable) value) { + array.add(convertFieldValue(item)); + } + } + json.put(field.getName(), array); + } else { + json.put(field.getName(), convertFieldValue(value)); + } + } + return json; + } + + private static Object convertFieldValue(Object value) { + if (value instanceof GeneratedMessageV3) { + return messageToJson((GeneratedMessageV3) value); + } + // 其他类型直接返回（Protobuf 基本类型可被 FastJSON 识别） + return value; + } + + public static void main(String[] args) { + + String filePath = "C:/Users/yuxin/Documents/xwechat_files/wxid_dtvj9sibla0d21_9cb3/msg/file/2026-02/public_info_data_1770264282958.todist"; + try { + // 1. 流式读取文件（避免大文件 OOM） + byte[] data = Files.readAllBytes(Paths.get(filePath)); + + // 2. Protobuf 反序列化 + EsOuterClass.EsSets esSets = EsOuterClass.EsSets.parseFrom(data); + System.out.println("✅ 成功解析 EsSets，共 " + esSets.getEsCount() + " 条记录"); + + // 3. 转换为 JSON（使用 Protobuf 原生 JsonFormat） + String json = protobufToJson(esSets); + + // 4. 输出格式化 JSON + System.out.println("/n📄 JSON Output:"); + System.out.println(json); + + } catch (InvalidProtocolBufferException e) { + System.err.println("❌ Protobuf 解析失败: " + e.getMessage()); + e.printStackTrace(); + } catch (IOException e) { + System.err.println("❌ 文件读取失败: " + e.getMessage()); + e.printStackTrace(); + } catch (Exception e) { + System.err.println("❌ 未知错误: " + e.getMessage()); + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/dsp/src/main/resources/application.yml b/dsp/src/main/resources/application.yml index 711ad4f..29640d3 100644 --- a/dsp/src/main/resources/application.yml +++ b/dsp/src/main/resources/application.yml @@ -63,7 +63,7 @@ switch: enable-storage-service: false enable-file-dl-service: false enable-protobuf-service: false - auto-export-and-upload: true + auto-export-and-upload: false ftp: host: 144.34.185.108