osc-group/osc
1
1
Fork 0
Code Pull Requests Projects Wiki Activity

针对6S采集的针对性改造

Browse Source
This commit is contained in:
yuxin-pc 2026-01-19 09:16:46 +08:00
parent 79b2d0d20a
commit ee958357b0
5 changed files with 21 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dsp/src/main/java/com/jsc/dsp/controller/ExportController.java
View File

@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSONObject;
import com.jsc.dsp.model.ReturnT; import com.jsc.dsp.model.ReturnT;
import com.jsc.dsp.utils.AutoExportAndUpload; import com.jsc.dsp.utils.AutoExportAndUpload;
import com.jsc.dsp.utils.DatabaseConnector; import com.jsc.dsp.utils.DatabaseConnector;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
@ -13,6 +14,7 @@ import javax.annotation.Resource;
@RestController @RestController
@RequestMapping("/export") @RequestMapping("/export")
@ConditionalOnProperty(name = "switch.auto-export-and-upload", havingValue = "true", matchIfMissing = true)
public class ExportController { public class ExportController {
@Resource @Resource

2
dsp/src/main/java/com/jsc/dsp/model/EsDataNewsView.java
View File

@ -24,10 +24,12 @@ public class EsDataNewsView {
String esSitename; String esSitename;
String esSrcname; String esSrcname;
String esUrlcontent; String esUrlcontent;
String esUrlcontentRaw;
String esUrlimage; String esUrlimage;
String esUrlname; String esUrlname;
String esUrltime; String esUrltime;
String esUrltitle; String esUrltitle;
String esUrltitleRaw;
String esAbstract; String esAbstract;
String esKeywords; String esKeywords;
String file; String file;

2
dsp/src/main/java/com/jsc/dsp/utils/AutoExportAndUpload.java
View File

@ -4,6 +4,7 @@ import com.jsc.dsp.service.ConfigService;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@ -28,6 +29,7 @@ import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream; import java.util.zip.ZipOutputStream;
@Component @Component
@ConditionalOnProperty(name = "switch.auto-export-and-upload", havingValue = "true", matchIfMissing = true)
public class AutoExportAndUpload { public class AutoExportAndUpload {
@Resource @Resource

20
dsp/src/main/java/com/jsc/dsp/utils/DatabaseConnector.java
View File

@ -116,18 +116,24 @@ public class DatabaseConnector {
row.createCell(5).setCellValue(item.getEsDoclength()); row.createCell(5).setCellValue(item.getEsDoclength());
row.createCell(6).setCellValue(item.getEsLang()); row.createCell(6).setCellValue(item.getEsLang());
row.createCell(7).setCellValue(item.getEsLasttime()); row.createCell(7).setCellValue(item.getEsLasttime());
if (item.getEsLinks().length() > 10000) {
row.createCell(8).setCellValue(item.getEsLinks().substring(0, 10000));
} else {
row.createCell(8).setCellValue(item.getEsLinks()); row.createCell(8).setCellValue(item.getEsLinks());
}
row.createCell(9).setCellValue(item.getEsLoadtime()); row.createCell(9).setCellValue(item.getEsLoadtime());
row.createCell(10).setCellValue(item.getEsSitename()); row.createCell(10).setCellValue(item.getEsSitename());
row.createCell(11).setCellValue(item.getEsSrcname()); row.createCell(11).setCellValue(item.getEsSrcname());
row.createCell(12).setCellValue(item.getEsUrlcontent()); row.createCell(12).setCellValue(item.getEsUrlcontent());
row.createCell(13).setCellValue(item.getEsUrlimage()); row.createCell(13).setCellValue(item.getEsUrlcontentRaw());
row.createCell(14).setCellValue(item.getEsUrlname()); row.createCell(14).setCellValue(item.getEsUrlimage());
row.createCell(15).setCellValue(item.getEsUrltime()); row.createCell(15).setCellValue(item.getEsUrlname());
row.createCell(16).setCellValue(item.getEsUrltitle()); row.createCell(16).setCellValue(item.getEsUrltime());
row.createCell(17).setCellValue(item.getEsAbstract()); row.createCell(17).setCellValue(item.getEsUrltitle());
row.createCell(18).setCellValue(item.getEsKeywords()); row.createCell(18).setCellValue(item.getEsUrltitleRaw());
row.createCell(19).setCellValue(item.getFile()); row.createCell(19).setCellValue(item.getEsAbstract());
row.createCell(20).setCellValue(item.getEsKeywords());
row.createCell(21).setCellValue(item.getFile());
} }
logger.info("完成excel数据写入，共" + rowNum + "行"); logger.info("完成excel数据写入，共" + rowNum + "行");

1
dsp/src/main/resources/application.yml
View File

@ -63,6 +63,7 @@ switch:
enable-storage-service: false enable-storage-service: false
enable-file-dl-service: false enable-file-dl-service: false
enable-protobuf-service: false enable-protobuf-service: false
auto-export-and-upload: true
ftp: ftp:
host: 144.34.185.108 host: 144.34.185.108