rename

dsp/src/main/java/com/jsc/dsp/utils/ExportAndUploadUtils.java

@@ -4,15 +4,10 @@ import com.jsc.dsp.service.ConfigService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
 import javax.annotation.Resource;
-import java.io.File;
+import java.io.*;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -21,15 +16,19 @@ import java.nio.file.attribute.BasicFileAttributes;
 import java.nio.file.attribute.FileTime;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
 import java.util.Comparator;
 import java.util.Date;
+import java.util.List;
+import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
 @Component
-public class AutoExportAndUpload {
+public class ExportAndUploadUtils {
 
     @Resource
     DatabaseConnector databaseConnector;
@@ -37,6 +36,9 @@ public class AutoExportAndUpload {
     @Resource
     FTPConnector ftpConnector;
 
+    @Resource
+    SFTPConnector sftpConnector;
+
     @Resource
     ConfigService configService;
 
@@ -46,8 +48,11 @@ public class AutoExportAndUpload {
     private static final SimpleDateFormat sdf = new SimpleDateFormat(DATE_FORMAT);
 
 
-    @Value("${custom.excelOutputPath}")
+    @Value("${custom.newsExcelOutputPath}")
-    String excelOutputPath;
+    String newsExcelOutputPath;
+
+    @Value("${custom.twitterExcelOutputPath}")
+    String twitterExcelOutputPath;
 
     @Value("${custom.backupFilePath}")
     String backupFilePath;
@@ -61,7 +66,7 @@ public class AutoExportAndUpload {
     /**
      * 每周一、三、五的早上8点，执行导出数据的任务
      */
-    public void exportDataAndUpload() {
+    public void exportNewsDataAndUpload() {
         logger.info("开始导出excel和pdf数据...");
         String lastLoadTime = configService.getConfigValueByName("last_loadtime");
         String currentLoadTime = StringUtils.DateToString(new Date());
@@ -72,20 +77,21 @@ public class AutoExportAndUpload {
         String zipFileName = "data_news-" + timestamp + "-001.zip";
         String zipFileFullName = backupFilePath + File.separator + zipFileName;
         String remoteZipPath = ftpUploadPath + "/" + zipFileName;
-        zipAndUploadDirectory(excelOutputPath, zipFileFullName, remoteZipPath);
+        zipAndUploadDirectory(newsExcelOutputPath, zipFileFullName, remoteZipPath);
     }
 
-    public void exportTwitterDataAndUpload(String startTime) {
+    public void exportTwitterDataAndUpload() {
         logger.info("开始导出twitter excel数据...");
-//        String twitterLastLoadTime = configService.getConfigValueByName("twitter_last_loadtime");
+        String twitterLastLoadTime = configService.getConfigValueByName("twitter_last_loadtime");
         String currentLoadTime = StringUtils.DateToString(new Date());
         String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd"));
-        databaseConnector.twitterToXlsx(startTime);
+        databaseConnector.twitterToXlsx(twitterLastLoadTime);
+        unzipAndMoveVideosImages(twitterLastLoadTime, currentLoadTime);
         configService.setConfigValueByName("twitter_last_loadtime", currentLoadTime);
         String zipFileName = "data_twitter-" + timestamp + "-001.zip";
         String zipFileFullName = backupFilePath + File.separator + zipFileName;
         String remoteZipPath = ftpUploadPath + "/" + zipFileName;
-        zipAndUploadDirectory(excelOutputPath, zipFileFullName, remoteZipPath);
+        zipAndUploadDirectory(twitterExcelOutputPath, zipFileFullName, remoteZipPath);
     }
 
     /**
@@ -123,16 +129,16 @@ public class AutoExportAndUpload {
         }
 
         // 上传 ZIP 文件
-//        try (InputStream zipInputStream = Files.newInputStream(localZipFile)) {
+        try (InputStream zipInputStream = Files.newInputStream(localZipFile)) {
-//            boolean uploaded = ftpConnector.uploadFile(zipInputStream, remoteZipPath);
+            boolean uploaded = sftpConnector.uploadFile(zipInputStream, remoteZipPath);
-//            if (uploaded) {
+            if (uploaded) {
-//                logger.info("ZIP 文件上传成功 - 本地: {}, FTP: {}", localZipPath, remoteZipPath);
+                logger.info("ZIP 文件上传成功 - 本地: {}, FTP: {}", localZipPath, remoteZipPath);
-//            } else {
+            } else {
-//                logger.error("ZIP 文件上传失败 - FTP: {}", remoteZipPath);
+                logger.error("ZIP 文件上传失败 - FTP: {}", remoteZipPath);
-//            }
+            }
-//        } catch (IOException e) {
+        } catch (IOException e) {
-//            logger.error("读取本地 ZIP 文件失败: {}", localZipPath, e);
+            logger.error("读取本地 ZIP 文件失败: {}", localZipPath, e);
-//        }
+        }
 
         // 注意：此处不再删除 localZipFile，由调用方决定是否保留或清理
     }
@@ -181,6 +187,155 @@ public class AutoExportAndUpload {
         }
     }
 
+    /**
+     * 解压存档文件并移动视频/图片目录
+     *
+     * @param startTime 业务开始时间（格式：yyyy-MM-dd HH:mm:ss，实际未使用但保留接口兼容性）
+     * @param endTime   业务结束时间（格式：yyyy-MM-dd HH:mm:ss）
+     */
+    public void unzipAndMoveVideosImages(String startTime, String endTime) {
+        logger.info("开始处理存档文件: startTime={}, endTime={}", startTime, endTime);
+
+        try {
+            // 1. 计算endTime前一日日期
+            LocalDate archiveDate = parseEndDate(endTime).minusDays(1);
+            String dateStr = archiveDate.format(DateTimeFormatter.ISO_DATE); // yyyy-MM-dd
+
+            // 2. 构建存档目录路径: D:/data/dbzq_backup/{yyyy}/{yyyy-MM}/{yyyy-MM-dd}
+            String year = String.valueOf(archiveDate.getYear());
+            String yearMonth = archiveDate.format(DateTimeFormatter.ofPattern("yyyy-MM"));
+            Path archiveBaseDir = Paths.get("D:/data/dbzq_backup", year, yearMonth, dateStr);
+
+            if (!Files.exists(archiveBaseDir) || !Files.isDirectory(archiveBaseDir)) {
+                logger.error("存档目录不存在: {}", archiveBaseDir);
+                throw new FileNotFoundException("存档目录不存在: " + archiveBaseDir);
+            }
+            logger.info("使用存档目录: {}", archiveBaseDir);
+
+            // 3. 确保输出目录存在
+            Path outputDir = Paths.get(twitterExcelOutputPath);
+            Files.createDirectories(outputDir);
+            logger.info("输出目录: {}", outputDir);
+
+            // 4. 处理视频压缩包 (image_data_plane_*.tar.gz)
+            processArchiveFiles(
+                    archiveBaseDir,
+                    "image_data_plane_",
+                    "videos",
+                    outputDir
+            );
+
+            // 5. 处理图片压缩包 (image_data_ship_*.tar.gz)
+            processArchiveFiles(
+                    archiveBaseDir,
+                    "image_data_ship_",
+                    "images",
+                    outputDir
+            );
+
+            logger.info("存档文件处理完成: {}", dateStr);
+
+        } catch (Exception e) {
+            logger.error("存档处理失败 [endTime={}]", endTime, e);
+            throw new RuntimeException("存档处理异常: " + e.getMessage(), e);
+        }
+    }
+
+    /**
+     * 解析结束时间字符串（兼容多种常见格式）
+     */
+    private LocalDate parseEndDate(String endTime) {
+        // 尝试常见时间格式
+        String[] patterns = {
+                "yyyy-MM-dd HH:mm:ss",
+                "yyyy-MM-dd'T'HH:mm:ss",
+                "yyyy-MM-dd HH:mm",
+                "yyyy-MM-dd"
+        };
+
+        for (String pattern : patterns) {
+            try {
+                DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern);
+                return LocalDate.parse(endTime.substring(0, 10), DateTimeFormatter.ISO_DATE); // 直接取日期部分
+            } catch (Exception ignored) {
+                // 尝试下一种格式
+            }
+        }
+
+        // 最终尝试完整解析
+        try {
+            return LocalDate.parse(endTime.trim().split("\\s+")[0]); // 取日期部分
+        } catch (DateTimeParseException e) {
+            throw new IllegalArgumentException("无法解析 endTime 格式: " + endTime +
+                    "，支持格式: yyyy-MM-dd[ HH:mm:ss]");
+        }
+    }
+
+    /**
+     * 处理指定前缀的压缩包
+     *
+     * @param archiveDir    存档目录
+     * @param filePrefix    文件前缀 (如 "image_data_plane_")
+     * @param targetDirName 目标目录名 (如 "videos")
+     * @param outputDir     输出根目录
+     */
+    private void processArchiveFiles(Path archiveDir, String filePrefix,
+                                     String targetDirName, Path outputDir) throws IOException {
+        // 查找所有匹配的tar.gz文件
+        List<Path> tarFiles = Files.list(archiveDir)
+                .filter(path -> Files.isRegularFile(path)
+                        && path.getFileName().toString().startsWith(filePrefix)
+                        && path.getFileName().toString().endsWith(".tar.gz"))
+                .sorted() // 按文件名排序确保处理顺序
+                .collect(Collectors.toList());
+
+        if (tarFiles.isEmpty()) {
+            logger.warn("未找到 {} 开头的压缩包: {}", filePrefix, archiveDir);
+            return;
+        }
+
+        logger.info("找到 {} 个 {} 压缩包: {}", tarFiles.size(), filePrefix,
+                tarFiles.stream().map(Path::getFileName).collect(Collectors.toList()));
+
+        // 创建全局临时目录（用于合并所有压缩包内容）
+        Path tempMergeDir = Files.createTempDirectory("archive_merge_");
+        logger.debug("创建临时合并目录: {}", tempMergeDir);
+
+        try {
+            // 步骤1: 依次解压所有tar.gz到临时目录
+            int totalFiles = 0;
+            for (Path tarFile : tarFiles) {
+                logger.info("解压压缩包: {}", tarFile.getFileName());
+                totalFiles += FileUtils.extractTarGz(tarFile.toFile(), tempMergeDir.toFile());
+            }
+
+            if (totalFiles == 0) {
+                logger.warn("解压后未发现任何文件，跳过移动: {}", filePrefix);
+                return;
+            }
+            logger.info("共解压 {} 个文件到临时目录", totalFiles);
+
+            // 步骤2: 平铺移动所有文件到目标目录（不保留目录结构，同名覆盖）
+            Path targetPath = outputDir.resolve(targetDirName);
+            Files.createDirectories(targetPath); // 确保目标目录存在
+
+            int movedCount = FileUtils.flattenAndMoveFiles(tempMergeDir, targetPath);
+
+            logger.info("成功平铺移动 {} 个文件到: {}", movedCount, targetPath);
+
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            // 清理临时目录
+            try {
+                FileUtils.deleteDirectory(tempMergeDir);
+                logger.debug("已清理临时目录: {}", tempMergeDir);
+            } catch (Exception e) {
+                logger.warn("清理临时目录失败: {}", tempMergeDir, e);
+            }
+        }
+    }
+
     public void copyPagesFiles(String startTime, String endTime) {
         try {
             logger.info("开始复制PDF...");
@@ -196,7 +351,7 @@ public class AutoExportAndUpload {
             }
 
             // 目标目录：在 excelOutputPath 下创建 pdf 子目录
-            Path targetBaseDir = Paths.get(excelOutputPath);
+            Path targetBaseDir = Paths.get(newsExcelOutputPath);
             Path targetPdfDir = targetBaseDir.resolve("pdf");
 
             // 确保目标目录存在