diff --git a/dsp/src/main/java/com/jsc/dsp/TestController.java b/dsp/src/main/java/com/jsc/dsp/controller/ExportController.java similarity index 78% rename from dsp/src/main/java/com/jsc/dsp/TestController.java rename to dsp/src/main/java/com/jsc/dsp/controller/ExportController.java index 3114015..dd65dcb 100644 --- a/dsp/src/main/java/com/jsc/dsp/TestController.java +++ b/dsp/src/main/java/com/jsc/dsp/controller/ExportController.java @@ -1,8 +1,8 @@ -package com.jsc.dsp; +package com.jsc.dsp.controller; import com.alibaba.fastjson.JSONObject; import com.jsc.dsp.model.ReturnT; -import com.jsc.dsp.utils.AutoPatroller; +import com.jsc.dsp.utils.AutoExportAndUpload; import com.jsc.dsp.utils.DatabaseConnector; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; @@ -12,14 +12,14 @@ import org.springframework.web.bind.annotation.RestController; import javax.annotation.Resource; @RestController -@RequestMapping("/test") -public class TestController { +@RequestMapping("/export") +public class ExportController { @Resource DatabaseConnector databaseConnector; @Resource - AutoPatroller autoPatroller; + AutoExportAndUpload autoExportAndUpload; @PostMapping("/exportExcel") public ReturnT exportExcel(@RequestBody JSONObject object) { @@ -32,10 +32,10 @@ public class TestController { } } - @PostMapping("/triggerExportTask") + @PostMapping("/triggerTask") public ReturnT triggerTask() { try { - autoPatroller.exportDataAndUpload(); + new Thread(() -> autoExportAndUpload.exportDataAndUpload()).start(); return new ReturnT<>(200, "", ""); } catch (Exception e) { return new ReturnT<>(500, e.getMessage(), ""); diff --git a/dsp/src/main/java/com/jsc/dsp/utils/AutoPatroller.java b/dsp/src/main/java/com/jsc/dsp/utils/AutoExportAndUpload.java similarity index 64% rename from dsp/src/main/java/com/jsc/dsp/utils/AutoPatroller.java rename to dsp/src/main/java/com/jsc/dsp/utils/AutoExportAndUpload.java index 8809918..1265ab1 100644 --- a/dsp/src/main/java/com/jsc/dsp/utils/AutoPatroller.java +++ b/dsp/src/main/java/com/jsc/dsp/utils/AutoExportAndUpload.java @@ -15,17 +15,20 @@ import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileTime; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.Comparator; import java.util.Date; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @Component -public class AutoPatroller { +public class AutoExportAndUpload { @Resource DatabaseConnector databaseConnector; @@ -54,14 +57,19 @@ public class AutoPatroller { @Value("${custom.ftpUploadPath}") String ftpUploadPath; - @Scheduled(cron = "0 0 8 * * *") + /** + * 每周一、三、五的早上8点，执行导出数据的任务 + */ + @Scheduled(cron = "${custom.exportTaskSchedule}") public void exportDataAndUpload() { + logger.info("开始导出excel和pdf数据..."); String lastLoadTime = configService.getConfigValueByName("last_loadtime"); String currentLoadTime = StringUtils.DateToString(new Date()); + String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd")); databaseConnector.exportToXlsx(lastLoadTime); - packagePagesFiles(lastLoadTime, currentLoadTime); + copyPagesFiles(lastLoadTime, currentLoadTime); configService.setConfigValueByName("last_loadtime", currentLoadTime); - String zipFileName = String.format("data_news-%s.zip", currentLoadTime.replace("-", "").replace(":", "").replace(" ", "")); + String zipFileName = "data_news-" + timestamp + "-001.zip"; String zipFileFullName = backupFilePath + File.separator + zipFileName; String remoteZipPath = ftpUploadPath + "/" + zipFileName; zipAndUploadDirectory(excelOutputPath, zipFileFullName, remoteZipPath); @@ -73,13 +81,12 @@ public class AutoPatroller { * @param sourceDirPath 本地要打包的源目录路径（如：/data/reports） * @param localZipPath 本地 ZIP 文件保存路径（如：/backup/archives/reports_20251224.zip） * @param remoteZipPath FTP 上的目标路径（如：/ftp/backups/reports_20251224.zip） - * @return 是否上传成功 */ - public boolean zipAndUploadDirectory(String sourceDirPath, String localZipPath, String remoteZipPath) { + public void zipAndUploadDirectory(String sourceDirPath, String localZipPath, String remoteZipPath) { Path sourceDir = Paths.get(sourceDirPath); if (!Files.exists(sourceDir) || !Files.isDirectory(sourceDir)) { logger.error("源目录不存在或不是一个目录: {}", sourceDirPath); - return false; + return; } Path localZipFile = Paths.get(localZipPath); @@ -90,7 +97,7 @@ public class AutoPatroller { logger.debug("创建 ZIP 父目录: {}", zipParent); } catch (IOException e) { logger.error("无法创建 ZIP 父目录: {}", zipParent, e); - return false; + return; } } @@ -99,7 +106,7 @@ public class AutoPatroller { zipDirectory(sourceDir, localZipFile.toFile()); } catch (IOException e) { logger.error("打包目录失败: {}", sourceDirPath, e); - return false; + return; } // 上传 ZIP 文件 @@ -110,10 +117,8 @@ public class AutoPatroller { } else { logger.error("ZIP 文件上传失败 - FTP: {}", remoteZipPath); } - return uploaded; } catch (IOException e) { logger.error("读取本地 ZIP 文件失败: {}", localZipPath, e); - return false; } // 注意：此处不再删除 localZipFile，由调用方决定是否保留或清理 @@ -163,72 +168,71 @@ public class AutoPatroller { } } - public void packagePagesFiles(String startTime, String endTime) { + public void copyPagesFiles(String startTime, String endTime) { try { + logger.info("开始复制PDF..."); // 解析时间范围 Date start = sdf.parse(startTime); Date end = sdf.parse(endTime); - // 确保输出目录存在 - Path excelOutputDir = Paths.get(excelOutputPath); - if (!Files.exists(excelOutputDir)) { - Files.createDirectories(excelOutputDir); - } - - // 构造 ZIP 文件名 - String zipFileName = String.format("pdf_files_%s_to_%s.zip", - startTime.replace(":", "").replace(" ", "_"), - endTime.replace(":", "").replace(" ", "_")); - Path zipFilePath = excelOutputDir.resolve(zipFileName); - - // 遍历 mhtmlOutputPath 目录 + // 源目录 Path sourceDir = Paths.get(pagesOutputPath); if (!Files.exists(sourceDir) || !Files.isDirectory(sourceDir)) { - System.err.println("源目录不存在或不是目录: " + pagesOutputPath); + logger.error("源目录不存在或不是目录: " + pagesOutputPath); return; } - try (ZipOutputStream zipOut = new ZipOutputStream(new FileOutputStream(zipFilePath.toFile()))) { - Files.walk(sourceDir) - .filter(path -> !Files.isDirectory(path)) - .filter(path -> path.toString().toLowerCase().endsWith(".pdf")) - .forEach(path -> { - try { - // 获取文件创建时间（Windows 支持，Linux/macOS 可能返回最早时间） - BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class); - FileTime creationTime = attrs.creationTime(); // 注意：非所有系统都支持创建时间 - Date fileCreationDate = new Date(creationTime.toMillis()); + // 目标目录：在 excelOutputPath 下创建 pdf 子目录 + Path targetBaseDir = Paths.get(excelOutputPath); + Path targetPdfDir = targetBaseDir.resolve("pdf"); - // 如果系统不支持创建时间（例如某些 Linux 系统），可改用 lastModifiedTime - // Date fileCreationDate = new Date(Files.getLastModifiedTime(path).toMillis()); - - if (!fileCreationDate.before(start) && !fileCreationDate.after(end)) { - // 文件创建时间在范围内 - String entryName = sourceDir.relativize(path).toString(); - zipOut.putNextEntry(new ZipEntry(entryName)); - try (InputStream in = Files.newInputStream(path)) { - byte[] buffer = new byte[8192]; - int len; - while ((len = in.read(buffer)) > 0) { - zipOut.write(buffer, 0, len); - } - } - zipOut.closeEntry(); - System.out.println("已添加文件: " + path); - } - } catch (IOException e) { - System.err.println("处理文件时出错: " + path + " - " + e.getMessage()); - } - }); + // 确保目标目录存在 + if (!Files.exists(targetPdfDir)) { + Files.createDirectories(targetPdfDir); } - System.out.println("ZIP 打包完成: " + zipFilePath); + // 遍历源目录中的所有 PDF 文件 + Files.walk(sourceDir) + .filter(path -> !Files.isDirectory(path)) + .filter(path -> path.toString().toLowerCase().endsWith(".pdf")) + .forEach(path -> { + try { + // 获取文件创建时间（注意：Linux/macOS 可能不支持 creationTime） + BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class); + FileTime creationTime = attrs.creationTime(); + Date fileCreationDate = new Date(creationTime.toMillis()); + + // 如果 creationTime 在某些系统上不可靠，可替换为 lastModifiedTime： + // Date fileCreationDate = new Date(Files.getLastModifiedTime(path).toMillis()); + + // 判断文件时间是否在指定范围内 + if (!fileCreationDate.before(start) && !fileCreationDate.after(end)) { + // 构建目标路径（保留相对结构，或直接放平？这里按原相对路径保留） + Path relativePath = sourceDir.relativize(path); + Path targetPath = targetPdfDir.resolve(relativePath); + + // 确保目标子目录存在 + Path targetParent = targetPath.getParent(); + if (targetParent != null && !Files.exists(targetParent)) { + Files.createDirectories(targetParent); + } + + // 复制文件 + Files.copy(path, targetPath, StandardCopyOption.REPLACE_EXISTING); + logger.info("已复制文件: " + path + " -> " + targetPath); + } + } catch (IOException e) { + logger.error("处理文件时出错: " + path + " - " + e.getMessage()); + } + }); + + logger.info("PDF 文件复制完成，目标目录: " + targetPdfDir.toAbsolutePath()); } catch (ParseException e) { - System.err.println("时间格式解析错误，请确保使用格式: " + DATE_FORMAT); + logger.error("时间格式解析错误，请确保使用格式: " + DATE_FORMAT); e.printStackTrace(); } catch (IOException e) { - System.err.println("IO 错误: " + e.getMessage()); + logger.error("IO 错误: " + e.getMessage()); e.printStackTrace(); } } diff --git a/dsp/src/main/java/com/jsc/dsp/utils/DatabaseConnector.java b/dsp/src/main/java/com/jsc/dsp/utils/DatabaseConnector.java index b37cdda..13ef2e3 100644 --- a/dsp/src/main/java/com/jsc/dsp/utils/DatabaseConnector.java +++ b/dsp/src/main/java/com/jsc/dsp/utils/DatabaseConnector.java @@ -6,11 +6,14 @@ import com.jsc.dsp.model.EsDataNewsView; import com.jsc.dsp.model.Indeximos; import org.apache.poi.ss.usermodel.*; import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import javax.annotation.Resource; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.lang.reflect.Field; @@ -20,7 +23,6 @@ import java.nio.file.Paths; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.List; -import java.util.logging.Logger; @Service public class DatabaseConnector { @@ -34,14 +36,14 @@ public class DatabaseConnector { @Value("${custom.excelOutputPath}") String excelOutputPath; - private final Logger logger = Logger.getLogger(this.getClass().getName()); + private final Logger logger = LoggerFactory.getLogger(this.getClass().getName()); public void insertIntoDB(List objectList) { try { indeximosRepository.saveAll(objectList); } catch (Exception e) { - logger.warning("Fail to insert data to Database"); - logger.warning(e.getMessage()); + logger.warn("Fail to insert data to Database"); + logger.warn(e.getMessage()); } } @@ -70,8 +72,8 @@ public class DatabaseConnector { if (!Files.exists(dirPath)) { Files.createDirectories(dirPath); } - String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss")); - String fileName = "data_news-" + timestamp + ".xlsx"; + String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd")); + String fileName = "data_news-" + timestamp + "-001.xlsx"; Path filePath = dirPath.resolve(fileName); List esDataNewsViewList = esDataNewsRepository.findAllByEsLoadtimeAfter(startTime); @@ -99,8 +101,13 @@ public class DatabaseConnector { for (EsDataNewsView item : esDataNewsViewList) { if (item.getFile() == null || item.getFile().length() < 5) { continue; + } else { + String fileFullPath = item.getFile(); + int i = fileFullPath.indexOf(File.separator); + item.setFile(fileFullPath.substring(i + 1)); } Row row = sheet.createRow(rowNum++); + logger.debug("导出excel第" + rowNum + "行"); row.createCell(0).setCellValue(item.getEsSid()); row.createCell(1).setCellValue(item.getEsAuthors()); row.createCell(2).setCellValue(item.getEsCarriertype()); @@ -122,6 +129,7 @@ public class DatabaseConnector { row.createCell(18).setCellValue(item.getEsKeywords()); row.createCell(19).setCellValue(item.getFile()); } + logger.info("完成excel数据写入，共" + rowNum + "行"); // 自动调整列宽 for (int i = 0; i < fields.length; i++) { @@ -137,6 +145,7 @@ public class DatabaseConnector { e.printStackTrace(); } } + logger.info("excel导出完成！"); } catch (Exception e) { e.printStackTrace(); } diff --git a/dsp/src/main/resources/application.yml b/dsp/src/main/resources/application.yml index ebcacb0..e47458a 100644 --- a/dsp/src/main/resources/application.yml +++ b/dsp/src/main/resources/application.yml @@ -1,5 +1,7 @@ server: port: 8084 + servlet: + context-path: /dsp spring: cloud: stream: @@ -88,4 +90,5 @@ custom: excelOutputPath: D:/data/output/upload backupFilePath: D:/data/output/backup pagesOutputPath: D:/data/output/pdf - ftpUploadPath: /home/jsc-2b \ No newline at end of file + ftpUploadPath: /home/jsc-2b + exportTaskSchedule: "0 0 12 * * 1,3,5" \ No newline at end of file