osc-group/osc
1
1
Fork 0
Code Pull Requests Projects Wiki Activity

[热搜] 热搜功能优化，excel导出及解压缩

Browse Source
This commit is contained in:
DELL 2026-03-02 13:49:42 +08:00
parent e1893f0ae3
commit 85158d00db
8 changed files with 96 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
dsp/src/main/java/com/jsc/dsp/controller/ExportController.java
View File

@ -43,6 +43,17 @@ public class ExportController {
}
}
@PostMapping("/exportHotSearchExcel")
public ReturnT<String> exportHotSearchExcel(@RequestBody JSONObject object) {
try {
String startTime = object.getString("startTime");
databaseConnector.hotSearchToXlsx(startTime);
return new ReturnT<>(200, "", "");
} catch (Exception e) {
return new ReturnT<>(500, e.getMessage(), "");
}
}
@PostMapping("/triggerTask")
public ReturnT<String> triggerTask() {
try {
@ -53,18 +64,8 @@ public class ExportController {
}
}
@PostMapping("/triggerTwitterTask")
public ReturnT<String> triggerTwitterTask() {
try {
new Thread(() -> exportAndUploadUtils.exportTwitterDataAndUpload()).start();
return new ReturnT<>(200, "", "");
} catch (Exception e) {
return new ReturnT<>(500, e.getMessage(), "");
}
}
@PostMapping("/triggerBaiduHotSeachTask")
public ReturnT<String> triggerBaiduHotSeachTask() {
@PostMapping("/triggerHotSearchTask")
public ReturnT<String> triggerHotSearchTask() {
try {
new Thread(() -> exportAndUploadUtils.exportHotSearchAndUpload()).start();
return new ReturnT<>(200, "", "");
@ -73,6 +74,4 @@ public class ExportController {
}
}
}

12
dsp/src/main/java/com/jsc/dsp/dao/EsDataBaiduRepository.java
View File

@ -1,12 +0,0 @@
package com.jsc.dsp.dao;
import com.jsc.dsp.model.EsDataBaiduView;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public interface EsDataBaiduRepository extends JpaRepository<EsDataBaiduView, String> {
List<EsDataBaiduView> findAllByEsLoadtimeAfter(String loadtime);
}

12
dsp/src/main/java/com/jsc/dsp/dao/EsDataHotSearchRepository.java Normal file
View File

@ -0,0 +1,12 @@
package com.jsc.dsp.dao;
import com.jsc.dsp.model.EsDataHotSearchView;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public interface EsDataHotSearchRepository extends JpaRepository<EsDataHotSearchView, String> {
List<EsDataHotSearchView> findAllByEsLoadtimeAfter(String loadtime);
}

15
dsp/src/main/java/com/jsc/dsp/model/EsDataBaiduView.java → dsp/src/main/java/com/jsc/dsp/model/EsDataHotSearchView.java
View File

@ -8,30 +8,31 @@ import javax.persistence.Table;
@Entity
@Data
@Table(name = "es_data_baidu")
public class EsDataBaiduView {
@Table(name = "es_data_hot_search")
public class EsDataHotSearchView {
@Id
private String esSid;
private String esUrltime;
private String esLoadtime;
private String esCarriertype;
private String esSitename;
private String esSimrank;
private String esUrltitle;
private String esUrlcontent;
private String esUrlname;
private String esUrltitle;
private String esHkey;
private String esLasttime;
private String esSimrank;
private String esHeat;
private String esLoadtime;
}

2
dsp/src/main/java/com/jsc/dsp/model/EsDataNewsView.java
View File

@ -33,4 +33,6 @@ public class EsDataNewsView {
String esAbstract;
String esKeywords;
String file;
String esHkey;
String esUrltopic;
}

96
dsp/src/main/java/com/jsc/dsp/utils/DatabaseConnector.java
View File

@ -1,11 +1,11 @@
package com.jsc.dsp.utils;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jsc.dsp.dao.EsDataBaiduRepository;
import com.jsc.dsp.dao.EsDataHotSearchRepository;
import com.jsc.dsp.dao.EsDataNewsRepository;
import com.jsc.dsp.dao.EsDataTwitterRepository;
import com.jsc.dsp.dao.IndeximosRepository;
import com.jsc.dsp.model.EsDataBaiduView;
import com.jsc.dsp.model.EsDataHotSearchView;
import com.jsc.dsp.model.EsDataNewsView;
import com.jsc.dsp.model.EsDataTwitterView;
import com.jsc.dsp.model.Indeximos;
@ -46,7 +46,7 @@ public class DatabaseConnector {
EsDataTwitterRepository esDataTwitterRepository;
@Resource
EsDataBaiduRepository esDataBaiduRepository;
EsDataHotSearchRepository esDataHotSearchRepository;
@Value("${custom.newsExcelOutputPath}")
String newsExcelOutputPath;
@ -54,8 +54,8 @@ public class DatabaseConnector {
@Value("${custom.twitterExcelOutputPath}")
String twitterExcelOutputPath;
@Value("${custom.baiduExcelOutputPath}")
String baiduExcelOutputPath;
@Value("${custom.hotSearchExcelOutputPath}")
String hotSearchExcelOutputPath;
private static final ObjectMapper objectMapper = new ObjectMapper();
@ -165,6 +165,8 @@ public class DatabaseConnector {
row.createCell(19).setCellValue(item.getEsAbstract());
row.createCell(20).setCellValue(item.getEsKeywords());
row.createCell(21).setCellValue(item.getFile());
row.createCell(22).setCellValue(item.getEsHkey());
row.createCell(23).setCellValue(item.getEsUrltopic());
}
logger.info("完成excel数据写入，共" + rowNum + "行");
@ -181,8 +183,8 @@ public class DatabaseConnector {
} catch (IOException e) {
e.printStackTrace();
}
}
logger.info("excel导出完成！");
logger.info("excel导出完成！");
} else logger.info("获取数据为空，excel未导出");
} catch (Exception e) {
e.printStackTrace();
}
@ -322,8 +324,8 @@ public class DatabaseConnector {
} catch (IOException e) {
e.printStackTrace();
}
}
logger.info("excel导出完成！");
logger.info("excel导出完成！");
} else logger.info("获取数据为空，excel未导出");
} catch (Exception e) {
e.printStackTrace();
}
@ -369,19 +371,19 @@ public class DatabaseConnector {
}
}
public void baiduToXlsx(String startTime) {
public void hotSearchToXlsx(String startTime) {
try {
Path dirPath = Paths.get(baiduExcelOutputPath);
Path dirPath = Paths.get(hotSearchExcelOutputPath);
if (!Files.exists(dirPath)) {
Files.createDirectories(dirPath);
}
String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd"));
String fileName = "data_baidu-" + timestamp + "-001.xlsx";
String fileName = "data_hot_search-" + timestamp + "-001.xlsx";
Path filePath = dirPath.resolve(fileName);
List<EsDataBaiduView> esDataNewsViewList = esDataBaiduRepository.findAllByEsLoadtimeAfter(startTime);
if (!esDataNewsViewList.isEmpty()) {
Field[] fields = esDataNewsViewList.get(0).getClass().getDeclaredFields(); // 通过反射获取类的成员信息，并使用这些类成员为后续生成的excel表头做准备
List<EsDataHotSearchView> esDataHotSearchViewList = esDataHotSearchRepository.findAllByEsLoadtimeAfter(startTime);
if (!esDataHotSearchViewList.isEmpty()) {
Field[] fields = esDataHotSearchViewList.get(0).getClass().getDeclaredFields(); // 通过反射获取类的成员信息，并使用这些类成员为后续生成的excel表头做准备
try (Workbook workbook = new XSSFWorkbook();
ByteArrayOutputStream out = new ByteArrayOutputStream()) {
@ -401,50 +403,56 @@ public class DatabaseConnector {
}
// 填充数据
int rowNum = 1;
for (EsDataBaiduView item : esDataNewsViewList) {
for (EsDataHotSearchView item : esDataHotSearchViewList) {
Row row = sheet.createRow(rowNum++);
logger.debug("导出excel第" + rowNum + "行");
// 0: esUrltime
row.createCell(0).setCellValue(item.getEsUrltime() != null ? item.getEsUrltime() : "");
// 0: esSid
row.createCell(0).setCellValue(item.getEsSid() != null ? item.getEsSid() : "");
// 1: esUrltime
row.createCell(1).setCellValue(item.getEsUrltime() != null ? item.getEsUrltime() : "");
// 1: esCarriertype
// 2: esCarriertype
row.createCell(2).setCellValue(item.getEsCarriertype() != null ? item.getEsCarriertype() : "");
// 2: esSitename
// 3: esSitename
row.createCell(3).setCellValue(item.getEsSitename() != null ? item.getEsSitename() : "");
// 3: esUrlcontent
String esUrlcontent = item.getEsUrlcontent();
if (esUrlcontent != null && esUrlcontent.length() > 10000) {
row.createCell(4).setCellValue(esUrlcontent.substring(0, 10000));
} else {
row.createCell(4).setCellValue(esUrlcontent != null ? esUrlcontent : "");
}
// 4: esUrlname
row.createCell(6).setCellValue(item.getEsUrlname() != null ? item.getEsUrlname() : "");
// 4: esSimrank
row.createCell(4).setCellValue(item.getEsSimrank() != null ? String.valueOf(Float.valueOf(item.getEsSimrank()).intValue()) : "");
// 5: esUrltitle
String esUrltitle = item.getEsUrltitle();
if (esUrltitle != null && esUrltitle.length() > 10000) {
row.createCell(7).setCellValue(esUrltitle.substring(0, 10000));
row.createCell(5).setCellValue(esUrltitle.substring(0, 10000));
} else {
row.createCell(7).setCellValue(esUrltitle != null ? esUrltitle : "");
row.createCell(5).setCellValue(esUrltitle != null ? esUrltitle : "");
}
// 6: esHkey
row.createCell(16).setCellValue(item.getEsHkey() != null ? item.getEsHkey() : "");
// 6: esUrlcontent
String esUrlcontent = item.getEsUrlcontent();
if (esUrlcontent != null && esUrlcontent.length() > 10000) {
row.createCell(6).setCellValue(esUrlcontent.substring(0, 10000));
} else {
row.createCell(6).setCellValue(esUrlcontent != null ? esUrlcontent : "");
}
// 7: esLasttime
// 7: esUrlname
row.createCell(7).setCellValue(item.getEsUrlname() != null ? item.getEsUrlname() : "");
// 8: esHkey
row.createCell(8).setCellValue(item.getEsHkey() != null ? item.getEsHkey() : "");
// 9: esLasttime
String esLasttime = extractFilenamesFromJsonArray(item.getEsLasttime());
row.createCell(17).setCellValue(esLasttime);
row.createCell(9).setCellValue(esLasttime);
// 8: esSimrank
row.createCell(18).setCellValue(item.getEsSimrank() != null ? item.getEsSimrank() : "");
// 9: esHeat
row.createCell(19).setCellValue(item.getEsHeat() != null ? item.getEsHeat() : "");
// 10: esHeat
row.createCell(10).setCellValue(item.getEsHeat() != null ? item.getEsHeat() : "");
// 1: esLasttime
String esLoadtime = extractFilenamesFromJsonArray(item.getEsLoadtime());
row.createCell(11).setCellValue(esLoadtime);
}
logger.info("完成excel数据写入，共" + rowNum + "行");
@ -461,10 +469,12 @@ public class DatabaseConnector {
} catch (IOException e) {
e.printStackTrace();
}
}
logger.info("excel导出完成！");
logger.info("excel导出完成！");
} else logger.info("获取数据为空，excel未导出");
} catch (Exception e) {
e.printStackTrace();
}
}
}
}

12
dsp/src/main/java/com/jsc/dsp/utils/ExportAndUploadUtils.java
View File

@ -54,8 +54,8 @@ public class ExportAndUploadUtils {
@Value("${custom.twitterExcelOutputPath}")
String twitterExcelOutputPath;
@Value("${custom.baiduExcelOutputPath}")
String baiduExcelOutputPath;
@Value("${custom.hotSearchExcelOutputPath}")
String hotSearchExcelOutputPath;
@Value("${custom.backupFilePath}")
String backupFilePath;
@ -99,15 +99,15 @@ public class ExportAndUploadUtils {
public void exportHotSearchAndUpload() {
logger.info("开始导出百度热搜 excel数据...");
String baiduLastLoadTime = configService.getConfigValueByName("baidu_last_loadtime");
String hotSearchLastLoadTime = configService.getConfigValueByName("hot_search_last_loadtime");
String currentLoadTime = StringUtils.DateToString(new Date());
String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd"));
databaseConnector.baiduToXlsx(baiduLastLoadTime);
configService.setConfigValueByName("baidu_hot_last_loadtime", currentLoadTime);
databaseConnector.hotSearchToXlsx(hotSearchLastLoadTime);
configService.setConfigValueByName("hot_search_last_loadtime", currentLoadTime);
String zipFileName = "data_hot_search-" + timestamp + "-001.zip";
String zipFileFullName = backupFilePath + File.separator + zipFileName;
String remoteZipPath = ftpUploadPath + "/" + zipFileName;
zipAndUploadDirectory(baiduExcelOutputPath, zipFileFullName, remoteZipPath);
zipAndUploadDirectory(hotSearchExcelOutputPath, zipFileFullName, remoteZipPath);
}
/**

4
dsp/src/main/resources/application.yml
View File

@ -1,5 +1,5 @@
server:
port: 8084
port: 18084
servlet:
context-path: /dsp
spring:
@ -98,7 +98,7 @@ custom:
twitterWhiteList: nytchinese;YesterdayBigcat;takaichi_sanae;yonhapcn;VOAChinese;ChineseWSJ;whyyoutouzhele;Jaemyung_Lee
newsExcelOutputPath: D:/data/output/upload
twitterExcelOutputPath: D:/data/output/twitter
baiduExcelOutputPath: D:/data/output/baidu
hotSearchExcelOutputPath: D:/data/output/hotSearch
backupFilePath: D:/data/output/backup
pagesOutputPath: D:/data/output/pdf
ftpUploadPath: /home/jsc-2b