From 15819c72607f8c1726c3edf0ab1f7bffb963fb6c Mon Sep 17 00:00:00 2001 From: yuxin-pc Date: Thu, 12 Feb 2026 08:58:33 +0800 Subject: [PATCH] rename --- ...dUpload.java => ExportAndUploadUtils.java} | 207 +++++++++++++++--- 1 file changed, 181 insertions(+), 26 deletions(-) rename dsp/src/main/java/com/jsc/dsp/utils/{AutoExportAndUpload.java => ExportAndUploadUtils.java} (56%) diff --git a/dsp/src/main/java/com/jsc/dsp/utils/AutoExportAndUpload.java b/dsp/src/main/java/com/jsc/dsp/utils/ExportAndUploadUtils.java similarity index 56% rename from dsp/src/main/java/com/jsc/dsp/utils/AutoExportAndUpload.java rename to dsp/src/main/java/com/jsc/dsp/utils/ExportAndUploadUtils.java index 06e7020..fefc5dd 100644 --- a/dsp/src/main/java/com/jsc/dsp/utils/AutoExportAndUpload.java +++ b/dsp/src/main/java/com/jsc/dsp/utils/ExportAndUploadUtils.java @@ -4,15 +4,10 @@ import com.jsc.dsp.service.ConfigService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import javax.annotation.Resource; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -21,15 +16,19 @@ import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileTime; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.LocalDate; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.Comparator; import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @Component -public class AutoExportAndUpload { +public class ExportAndUploadUtils { @Resource DatabaseConnector databaseConnector; @@ -37,6 +36,9 @@ public class AutoExportAndUpload { @Resource FTPConnector ftpConnector; + @Resource + SFTPConnector sftpConnector; + @Resource ConfigService configService; @@ -46,8 +48,11 @@ public class AutoExportAndUpload { private static final SimpleDateFormat sdf = new SimpleDateFormat(DATE_FORMAT); - @Value("${custom.excelOutputPath}") - String excelOutputPath; + @Value("${custom.newsExcelOutputPath}") + String newsExcelOutputPath; + + @Value("${custom.twitterExcelOutputPath}") + String twitterExcelOutputPath; @Value("${custom.backupFilePath}") String backupFilePath; @@ -61,7 +66,7 @@ public class AutoExportAndUpload { /** * 每周一、三、五的早上8点，执行导出数据的任务 */ - public void exportDataAndUpload() { + public void exportNewsDataAndUpload() { logger.info("开始导出excel和pdf数据..."); String lastLoadTime = configService.getConfigValueByName("last_loadtime"); String currentLoadTime = StringUtils.DateToString(new Date()); @@ -72,20 +77,21 @@ public class AutoExportAndUpload { String zipFileName = "data_news-" + timestamp + "-001.zip"; String zipFileFullName = backupFilePath + File.separator + zipFileName; String remoteZipPath = ftpUploadPath + "/" + zipFileName; - zipAndUploadDirectory(excelOutputPath, zipFileFullName, remoteZipPath); + zipAndUploadDirectory(newsExcelOutputPath, zipFileFullName, remoteZipPath); } - public void exportTwitterDataAndUpload(String startTime) { + public void exportTwitterDataAndUpload() { logger.info("开始导出twitter excel数据..."); -// String twitterLastLoadTime = configService.getConfigValueByName("twitter_last_loadtime"); + String twitterLastLoadTime = configService.getConfigValueByName("twitter_last_loadtime"); String currentLoadTime = StringUtils.DateToString(new Date()); String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd")); - databaseConnector.twitterToXlsx(startTime); + databaseConnector.twitterToXlsx(twitterLastLoadTime); + unzipAndMoveVideosImages(twitterLastLoadTime, currentLoadTime); configService.setConfigValueByName("twitter_last_loadtime", currentLoadTime); String zipFileName = "data_twitter-" + timestamp + "-001.zip"; String zipFileFullName = backupFilePath + File.separator + zipFileName; String remoteZipPath = ftpUploadPath + "/" + zipFileName; - zipAndUploadDirectory(excelOutputPath, zipFileFullName, remoteZipPath); + zipAndUploadDirectory(twitterExcelOutputPath, zipFileFullName, remoteZipPath); } /** @@ -123,16 +129,16 @@ public class AutoExportAndUpload { } // 上传 ZIP 文件 -// try (InputStream zipInputStream = Files.newInputStream(localZipFile)) { -// boolean uploaded = ftpConnector.uploadFile(zipInputStream, remoteZipPath); -// if (uploaded) { -// logger.info("ZIP 文件上传成功 - 本地: {}, FTP: {}", localZipPath, remoteZipPath); -// } else { -// logger.error("ZIP 文件上传失败 - FTP: {}", remoteZipPath); -// } -// } catch (IOException e) { -// logger.error("读取本地 ZIP 文件失败: {}", localZipPath, e); -// } + try (InputStream zipInputStream = Files.newInputStream(localZipFile)) { + boolean uploaded = sftpConnector.uploadFile(zipInputStream, remoteZipPath); + if (uploaded) { + logger.info("ZIP 文件上传成功 - 本地: {}, FTP: {}", localZipPath, remoteZipPath); + } else { + logger.error("ZIP 文件上传失败 - FTP: {}", remoteZipPath); + } + } catch (IOException e) { + logger.error("读取本地 ZIP 文件失败: {}", localZipPath, e); + } // 注意：此处不再删除 localZipFile，由调用方决定是否保留或清理 } @@ -181,6 +187,155 @@ public class AutoExportAndUpload { } } + /** + * 解压存档文件并移动视频/图片目录 + * + * @param startTime 业务开始时间（格式：yyyy-MM-dd HH:mm:ss，实际未使用但保留接口兼容性） + * @param endTime 业务结束时间（格式：yyyy-MM-dd HH:mm:ss） + */ + public void unzipAndMoveVideosImages(String startTime, String endTime) { + logger.info("开始处理存档文件: startTime={}, endTime={}", startTime, endTime); + + try { + // 1. 计算endTime前一日日期 + LocalDate archiveDate = parseEndDate(endTime).minusDays(1); + String dateStr = archiveDate.format(DateTimeFormatter.ISO_DATE); // yyyy-MM-dd + + // 2. 构建存档目录路径: D:/data/dbzq_backup/{yyyy}/{yyyy-MM}/{yyyy-MM-dd} + String year = String.valueOf(archiveDate.getYear()); + String yearMonth = archiveDate.format(DateTimeFormatter.ofPattern("yyyy-MM")); + Path archiveBaseDir = Paths.get("D:/data/dbzq_backup", year, yearMonth, dateStr); + + if (!Files.exists(archiveBaseDir) || !Files.isDirectory(archiveBaseDir)) { + logger.error("存档目录不存在: {}", archiveBaseDir); + throw new FileNotFoundException("存档目录不存在: " + archiveBaseDir); + } + logger.info("使用存档目录: {}", archiveBaseDir); + + // 3. 确保输出目录存在 + Path outputDir = Paths.get(twitterExcelOutputPath); + Files.createDirectories(outputDir); + logger.info("输出目录: {}", outputDir); + + // 4. 处理视频压缩包 (image_data_plane_*.tar.gz) + processArchiveFiles( + archiveBaseDir, + "image_data_plane_", + "videos", + outputDir + ); + + // 5. 处理图片压缩包 (image_data_ship_*.tar.gz) + processArchiveFiles( + archiveBaseDir, + "image_data_ship_", + "images", + outputDir + ); + + logger.info("存档文件处理完成: {}", dateStr); + + } catch (Exception e) { + logger.error("存档处理失败 [endTime={}]", endTime, e); + throw new RuntimeException("存档处理异常: " + e.getMessage(), e); + } + } + + /** + * 解析结束时间字符串（兼容多种常见格式） + */ + private LocalDate parseEndDate(String endTime) { + // 尝试常见时间格式 + String[] patterns = { + "yyyy-MM-dd HH:mm:ss", + "yyyy-MM-dd'T'HH:mm:ss", + "yyyy-MM-dd HH:mm", + "yyyy-MM-dd" + }; + + for (String pattern : patterns) { + try { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern); + return LocalDate.parse(endTime.substring(0, 10), DateTimeFormatter.ISO_DATE); // 直接取日期部分 + } catch (Exception ignored) { + // 尝试下一种格式 + } + } + + // 最终尝试完整解析 + try { + return LocalDate.parse(endTime.trim().split("\\s+")[0]); // 取日期部分 + } catch (DateTimeParseException e) { + throw new IllegalArgumentException("无法解析 endTime 格式: " + endTime + + "，支持格式: yyyy-MM-dd[ HH:mm:ss]"); + } + } + + /** + * 处理指定前缀的压缩包 + * + * @param archiveDir 存档目录 + * @param filePrefix 文件前缀 (如 "image_data_plane_") + * @param targetDirName 目标目录名 (如 "videos") + * @param outputDir 输出根目录 + */ + private void processArchiveFiles(Path archiveDir, String filePrefix, + String targetDirName, Path outputDir) throws IOException { + // 查找所有匹配的tar.gz文件 + List tarFiles = Files.list(archiveDir) + .filter(path -> Files.isRegularFile(path) + && path.getFileName().toString().startsWith(filePrefix) + && path.getFileName().toString().endsWith(".tar.gz")) + .sorted() // 按文件名排序确保处理顺序 + .collect(Collectors.toList()); + + if (tarFiles.isEmpty()) { + logger.warn("未找到 {} 开头的压缩包: {}", filePrefix, archiveDir); + return; + } + + logger.info("找到 {} 个 {} 压缩包: {}", tarFiles.size(), filePrefix, + tarFiles.stream().map(Path::getFileName).collect(Collectors.toList())); + + // 创建全局临时目录（用于合并所有压缩包内容） + Path tempMergeDir = Files.createTempDirectory("archive_merge_"); + logger.debug("创建临时合并目录: {}", tempMergeDir); + + try { + // 步骤1: 依次解压所有tar.gz到临时目录 + int totalFiles = 0; + for (Path tarFile : tarFiles) { + logger.info("解压压缩包: {}", tarFile.getFileName()); + totalFiles += FileUtils.extractTarGz(tarFile.toFile(), tempMergeDir.toFile()); + } + + if (totalFiles == 0) { + logger.warn("解压后未发现任何文件，跳过移动: {}", filePrefix); + return; + } + logger.info("共解压 {} 个文件到临时目录", totalFiles); + + // 步骤2: 平铺移动所有文件到目标目录（不保留目录结构，同名覆盖） + Path targetPath = outputDir.resolve(targetDirName); + Files.createDirectories(targetPath); // 确保目标目录存在 + + int movedCount = FileUtils.flattenAndMoveFiles(tempMergeDir, targetPath); + + logger.info("成功平铺移动 {} 个文件到: {}", movedCount, targetPath); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + // 清理临时目录 + try { + FileUtils.deleteDirectory(tempMergeDir); + logger.debug("已清理临时目录: {}", tempMergeDir); + } catch (Exception e) { + logger.warn("清理临时目录失败: {}", tempMergeDir, e); + } + } + } + public void copyPagesFiles(String startTime, String endTime) { try { logger.info("开始复制PDF..."); @@ -196,7 +351,7 @@ public class AutoExportAndUpload { } // 目标目录：在 excelOutputPath 下创建 pdf 子目录 - Path targetBaseDir = Paths.get(excelOutputPath); + Path targetBaseDir = Paths.get(newsExcelOutputPath); Path targetPdfDir = targetBaseDir.resolve("pdf"); // 确保目标目录存在