osc-group/osc
1
1
Fork 0
Code Pull Requests Projects Wiki Activity

jsc-dsp #1

Merged
yuxin merged 9 commits from jsc-dsp into main 2026-01-19 01:30:58 +00:00
Conversation 1 Commits 9 Files Changed 37 +90 -74
4 changed files with 90 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 79b2d0d20a - Show all commits

14
dsp/src/main/java/com/jsc/dsp/TestController.java → dsp/src/main/java/com/jsc/dsp/controller/ExportController.java
View File

@ -1,8 +1,8 @@
package com.jsc.dsp;
package com.jsc.dsp.controller;
import com.alibaba.fastjson.JSONObject;
import com.jsc.dsp.model.ReturnT;
import com.jsc.dsp.utils.AutoPatroller;
import com.jsc.dsp.utils.AutoExportAndUpload;
import com.jsc.dsp.utils.DatabaseConnector;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
@ -12,14 +12,14 @@ import org.springframework.web.bind.annotation.RestController;
import javax.annotation.Resource;
@RestController
@RequestMapping("/test")
public class TestController {
@RequestMapping("/export")
public class ExportController {
@Resource
DatabaseConnector databaseConnector;
@Resource
AutoPatroller autoPatroller;
AutoExportAndUpload autoExportAndUpload;
@PostMapping("/exportExcel")
public ReturnT<String> exportExcel(@RequestBody JSONObject object) {
@ -32,10 +32,10 @@ public class TestController {
}
}
@PostMapping("/triggerExportTask")
@PostMapping("/triggerTask")
public ReturnT<String> triggerTask() {
try {
autoPatroller.exportDataAndUpload();
new Thread(() -> autoExportAndUpload.exportDataAndUpload()).start();
return new ReturnT<>(200, "", "");
} catch (Exception e) {
return new ReturnT<>(500, e.getMessage(), "");

124
dsp/src/main/java/com/jsc/dsp/utils/AutoPatroller.java → dsp/src/main/java/com/jsc/dsp/utils/AutoExportAndUpload.java
View File

@ -15,17 +15,20 @@ import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileTime;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Comparator;
import java.util.Date;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@Component
public class AutoPatroller {
public class AutoExportAndUpload {
@Resource
DatabaseConnector databaseConnector;
@ -54,14 +57,19 @@ public class AutoPatroller {
@Value("${custom.ftpUploadPath}")
String ftpUploadPath;
@Scheduled(cron = "0 0 8 * * *")
/**
* 每周一、三、五的早上8点，执行导出数据的任务
*/
@Scheduled(cron = "${custom.exportTaskSchedule}")
public void exportDataAndUpload() {
logger.info("开始导出excel和pdf数据...");
String lastLoadTime = configService.getConfigValueByName("last_loadtime");
String currentLoadTime = StringUtils.DateToString(new Date());
String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd"));
databaseConnector.exportToXlsx(lastLoadTime);
packagePagesFiles(lastLoadTime, currentLoadTime);
copyPagesFiles(lastLoadTime, currentLoadTime);
configService.setConfigValueByName("last_loadtime", currentLoadTime);
String zipFileName = String.format("data_news-%s.zip", currentLoadTime.replace("-", "").replace(":", "").replace(" ", ""));
String zipFileName = "data_news-" + timestamp + "-001.zip";
String zipFileFullName = backupFilePath + File.separator + zipFileName;
String remoteZipPath = ftpUploadPath + "/" + zipFileName;
zipAndUploadDirectory(excelOutputPath, zipFileFullName, remoteZipPath);
@ -73,13 +81,12 @@ public class AutoPatroller {
* @param sourceDirPath 本地要打包的源目录路径（如：/data/reports）
* @param localZipPath 本地 ZIP 文件保存路径（如：/backup/archives/reports_20251224.zip）
* @param remoteZipPath FTP 上的目标路径（如：/ftp/backups/reports_20251224.zip）
* @return 是否上传成功
*/
public boolean zipAndUploadDirectory(String sourceDirPath, String localZipPath, String remoteZipPath) {
public void zipAndUploadDirectory(String sourceDirPath, String localZipPath, String remoteZipPath) {
Path sourceDir = Paths.get(sourceDirPath);
if (!Files.exists(sourceDir) || !Files.isDirectory(sourceDir)) {
logger.error("源目录不存在或不是一个目录: {}", sourceDirPath);
return false;
return;
}
Path localZipFile = Paths.get(localZipPath);
@ -90,7 +97,7 @@ public class AutoPatroller {
logger.debug("创建 ZIP 父目录: {}", zipParent);
} catch (IOException e) {
logger.error("无法创建 ZIP 父目录: {}", zipParent, e);
return false;
return;
}
}
@ -99,7 +106,7 @@ public class AutoPatroller {
zipDirectory(sourceDir, localZipFile.toFile());
} catch (IOException e) {
logger.error("打包目录失败: {}", sourceDirPath, e);
return false;
return;
}
// 上传 ZIP 文件
@ -110,10 +117,8 @@ public class AutoPatroller {
} else {
logger.error("ZIP 文件上传失败 - FTP: {}", remoteZipPath);
}
return uploaded;
} catch (IOException e) {
logger.error("读取本地 ZIP 文件失败: {}", localZipPath, e);
return false;
}
// 注意：此处不再删除 localZipFile，由调用方决定是否保留或清理
@ -163,72 +168,71 @@ public class AutoPatroller {
}
}
public void packagePagesFiles(String startTime, String endTime) {
public void copyPagesFiles(String startTime, String endTime) {
try {
logger.info("开始复制PDF...");
// 解析时间范围
Date start = sdf.parse(startTime);
Date end = sdf.parse(endTime);
// 确保输出目录存在
Path excelOutputDir = Paths.get(excelOutputPath);
if (!Files.exists(excelOutputDir)) {
Files.createDirectories(excelOutputDir);
}
// 构造 ZIP 文件名
String zipFileName = String.format("pdf_files_%s_to_%s.zip",
startTime.replace(":", "").replace(" ", "_"),
endTime.replace(":", "").replace(" ", "_"));
Path zipFilePath = excelOutputDir.resolve(zipFileName);
// 遍历 mhtmlOutputPath 目录
// 源目录
Path sourceDir = Paths.get(pagesOutputPath);
if (!Files.exists(sourceDir) || !Files.isDirectory(sourceDir)) {
System.err.println("源目录不存在或不是目录: " + pagesOutputPath);
logger.error("源目录不存在或不是目录: " + pagesOutputPath);
return;
}
try (ZipOutputStream zipOut = new ZipOutputStream(new FileOutputStream(zipFilePath.toFile()))) {
Files.walk(sourceDir)
.filter(path -> !Files.isDirectory(path))
.filter(path -> path.toString().toLowerCase().endsWith(".pdf"))
.forEach(path -> {
try {
// 获取文件创建时间（Windows 支持，Linux/macOS 可能返回最早时间）
BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class);
FileTime creationTime = attrs.creationTime(); // 注意：非所有系统都支持创建时间
Date fileCreationDate = new Date(creationTime.toMillis());
// 目标目录：在 excelOutputPath 下创建 pdf 子目录
Path targetBaseDir = Paths.get(excelOutputPath);
Path targetPdfDir = targetBaseDir.resolve("pdf");
// 如果系统不支持创建时间（例如某些 Linux 系统），可改用 lastModifiedTime
// Date fileCreationDate = new Date(Files.getLastModifiedTime(path).toMillis());
if (!fileCreationDate.before(start) && !fileCreationDate.after(end)) {
// 文件创建时间在范围内
String entryName = sourceDir.relativize(path).toString();
zipOut.putNextEntry(new ZipEntry(entryName));
try (InputStream in = Files.newInputStream(path)) {
byte[] buffer = new byte[8192];
int len;
while ((len = in.read(buffer)) > 0) {
zipOut.write(buffer, 0, len);
}
}
zipOut.closeEntry();
System.out.println("已添加文件: " + path);
}
} catch (IOException e) {
System.err.println("处理文件时出错: " + path + " - " + e.getMessage());
}
});
// 确保目标目录存在
if (!Files.exists(targetPdfDir)) {
Files.createDirectories(targetPdfDir);
}
System.out.println("ZIP 打包完成: " + zipFilePath);
// 遍历源目录中的所有 PDF 文件
Files.walk(sourceDir)
.filter(path -> !Files.isDirectory(path))
.filter(path -> path.toString().toLowerCase().endsWith(".pdf"))
.forEach(path -> {
try {
// 获取文件创建时间（注意：Linux/macOS 可能不支持 creationTime）
BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class);
FileTime creationTime = attrs.creationTime();
Date fileCreationDate = new Date(creationTime.toMillis());
// 如果 creationTime 在某些系统上不可靠，可替换为 lastModifiedTime：
// Date fileCreationDate = new Date(Files.getLastModifiedTime(path).toMillis());
// 判断文件时间是否在指定范围内
if (!fileCreationDate.before(start) && !fileCreationDate.after(end)) {
// 构建目标路径（保留相对结构，或直接放平？这里按原相对路径保留）
Path relativePath = sourceDir.relativize(path);
Path targetPath = targetPdfDir.resolve(relativePath);
// 确保目标子目录存在
Path targetParent = targetPath.getParent();
if (targetParent != null && !Files.exists(targetParent)) {
Files.createDirectories(targetParent);
}
// 复制文件
Files.copy(path, targetPath, StandardCopyOption.REPLACE_EXISTING);
logger.info("已复制文件: " + path + " -> " + targetPath);
}
} catch (IOException e) {
logger.error("处理文件时出错: " + path + " - " + e.getMessage());
}
});
logger.info("PDF 文件复制完成，目标目录: " + targetPdfDir.toAbsolutePath());
} catch (ParseException e) {
System.err.println("时间格式解析错误，请确保使用格式: " + DATE_FORMAT);
logger.error("时间格式解析错误，请确保使用格式: " + DATE_FORMAT);
e.printStackTrace();
} catch (IOException e) {
System.err.println("IO 错误: " + e.getMessage());
logger.error("IO 错误: " + e.getMessage());
e.printStackTrace();
}
}

21
dsp/src/main/java/com/jsc/dsp/utils/DatabaseConnector.java
View File

@ -6,11 +6,14 @@ import com.jsc.dsp.model.EsDataNewsView;
import com.jsc.dsp.model.Indeximos;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.lang.reflect.Field;
@ -20,7 +23,6 @@ import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.logging.Logger;
@Service
public class DatabaseConnector {
@ -34,14 +36,14 @@ public class DatabaseConnector {
@Value("${custom.excelOutputPath}")
String excelOutputPath;
private final Logger logger = Logger.getLogger(this.getClass().getName());
private final Logger logger = LoggerFactory.getLogger(this.getClass().getName());
public void insertIntoDB(List<Indeximos> objectList) {
try {
indeximosRepository.saveAll(objectList);
} catch (Exception e) {
logger.warning("Fail to insert data to Database");
logger.warning(e.getMessage());
logger.warn("Fail to insert data to Database");
logger.warn(e.getMessage());
}
}
@ -70,8 +72,8 @@ public class DatabaseConnector {
if (!Files.exists(dirPath)) {
Files.createDirectories(dirPath);
}
String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss"));
String fileName = "data_news-" + timestamp + ".xlsx";
String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd"));
String fileName = "data_news-" + timestamp + "-001.xlsx";
Path filePath = dirPath.resolve(fileName);
List<EsDataNewsView> esDataNewsViewList = esDataNewsRepository.findAllByEsLoadtimeAfter(startTime);
@ -99,8 +101,13 @@ public class DatabaseConnector {
for (EsDataNewsView item : esDataNewsViewList) {
if (item.getFile() == null || item.getFile().length() < 5) {
continue;
} else {
String fileFullPath = item.getFile();
int i = fileFullPath.indexOf(File.separator);
item.setFile(fileFullPath.substring(i + 1));
}
Row row = sheet.createRow(rowNum++);
logger.debug("导出excel第" + rowNum + "行");
row.createCell(0).setCellValue(item.getEsSid());
row.createCell(1).setCellValue(item.getEsAuthors());
row.createCell(2).setCellValue(item.getEsCarriertype());
@ -122,6 +129,7 @@ public class DatabaseConnector {
row.createCell(18).setCellValue(item.getEsKeywords());
row.createCell(19).setCellValue(item.getFile());
}
logger.info("完成excel数据写入，共" + rowNum + "行");
// 自动调整列宽
for (int i = 0; i < fields.length; i++) {
@ -137,6 +145,7 @@ public class DatabaseConnector {
e.printStackTrace();
}
}
logger.info("excel导出完成！");
} catch (Exception e) {
e.printStackTrace();
}

5
dsp/src/main/resources/application.yml
View File

@ -1,5 +1,7 @@
server:
port: 8084
servlet:
context-path: /dsp
spring:
cloud:
stream:
@ -88,4 +90,5 @@ custom:
excelOutputPath: D:/data/output/upload
backupFilePath: D:/data/output/backup
pagesOutputPath: D:/data/output/pdf
ftpUploadPath: /home/jsc-2b
ftpUploadPath: /home/jsc-2b
exportTaskSchedule: "0 0 12 * * 1,3,5"