osc-group/osc
1
1
Fork 0
Code Pull Requests Projects Wiki Activity

jsc-dsp #1

Merged
yuxin merged 9 commits from jsc-dsp into main 2026-01-19 01:30:58 +00:00
Conversation 1 Commits 9 Files Changed 37 +23 -134
4 changed files with 23 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 4de86e5f40 - Show all commits

16
dsp/src/main/java/com/jsc/dsp/config/Configuration.java
View File

@ -16,18 +16,6 @@ import java.util.concurrent.Executors;
@Component
public class Configuration {
@Value("${es.ip}")
String esIp;
@Value("${es.port}")
Integer esPort;
@Value("${es.username}")
String esUsername;
@Value("${es.password}")
String esPassword;
@Bean
public JacksonJsonParser getJacksonParser() {
return new JacksonJsonParser();
@ -48,8 +36,4 @@ public class Configuration {
return Executors.newFixedThreadPool(4);
}
@Bean
public RestHighLevelClient esClient() {
return EsUtils.getElasticsearchClient(esIp, esPort, esUsername, esPassword);
}
}

23
dsp/src/main/java/com/jsc/dsp/service/FileDlService.java
View File

@ -6,6 +6,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.json.JacksonJsonParser;
import org.springframework.cloud.stream.annotation.EnableBinding;
import org.springframework.cloud.stream.annotation.StreamListener;
@ -21,6 +22,7 @@ import java.util.concurrent.Executors;
@Component
@EnableBinding(FileDlBinding.class)
@ConditionalOnProperty(name = "switch.enable-file-dl-service", havingValue = "true", matchIfMissing = true)
public class FileDlService extends StreamService {
@Autowired
@ -78,31 +80,14 @@ public class FileDlService extends StreamService {
}
int dlResult = fileUtils.downloadFromUrl(fileURL, protoSavePath);
if (dlResult == 1) {
File transferPath = new File(transferBackupPath);
File transferPath = new File(protoSavePath);
File[] files = transferPath.listFiles();
if (files != null && files.length > 0) {
for (File transferFile : files) {
if (transferFile.getName().endsWith(".tar.gz")) {
if (transferFile.getName().startsWith("attach")) {
try {
fileUtils.UnzipTarGzip(transferFile.getAbsolutePath(), nginxPath);
logger.info("Unzip attachments " + transferFile.getName());
} catch (Exception e) {
logger.error("Unzip error!");
}
} else {
try {
fileUtils.UnzipTarGzip(transferFile.getAbsolutePath(), fileUnzipPath);
logger.info("Unzip " + transferFile.getName());
} catch (Exception e) {
logger.error("Unzip error!");
}
}
fileUtils.moveFileToBackupFolder(transferFile, keepBackupFile);
}
fileUtils.moveFileToBackupFolder(transferFile, keepBackupFile);
}
// Runnable upload2Ceph = () -> fileUtils.uploadToCeph(fileUnzipPath);
// pool.execute(upload2Ceph);
}
} else if (dlResult == 0) {
logger.error("File " + fileName + " download failure");

10
dsp/src/main/java/com/jsc/dsp/service/ProtobufService.java
View File

@ -12,12 +12,14 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.json.JacksonJsonParser;
import org.springframework.cloud.stream.annotation.EnableBinding;
import org.springframework.cloud.stream.annotation.StreamListener;
import org.springframework.messaging.support.MessageBuilder;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStream;
@ -30,6 +32,7 @@ import java.util.Map;
@Component
@EnableBinding(ProtobufBinding.class)
@ConditionalOnProperty(name = "switch.enable-protobuf-service", havingValue = "true", matchIfMissing = true)
public class ProtobufService extends StreamService {
@Autowired
@ -41,9 +44,6 @@ public class ProtobufService extends StreamService {
@Value("${custom.proto_save_path}")
String protoSavePath;
@Value("${custom.transfer_backup_path}")
String transferBackupPath;
@Value("${custom.keep_backup_file}")
String keepBackupFile;
@ -55,7 +55,7 @@ public class ProtobufService extends StreamService {
private final Logger logger = LogManager.getLogger(ProtobufService.class.getName());
@Autowired
@Resource
private ProtobufBinding source;
@Override
@ -169,7 +169,7 @@ public class ProtobufService extends StreamService {
}
logger.debug("protobuf done");
// 转移备份目录的todist文件
File transferPath = new File(transferBackupPath);
File transferPath = new File(protoSavePath);
File[] files = transferPath.listFiles();
if (files != null && files.length > 0) {
for (File transferFile : files) {

108
dsp/src/main/java/com/jsc/dsp/utils/AutoPatroller.java
View File

@ -37,99 +37,19 @@ public class AutoPatroller {
@Value("${custom.socialUpdateAPI}")
String socialUpdateAPI;
@Value("${es.ip}")
String esIp;
@Value("${es.port}")
Integer esPort;
@Value("${es.username}")
String esUsername;
@Value("${es.password}")
String esPassword;
@Scheduled(cron = "0 45 0/3 * * *")
public void checkNewsSite() {
checkWebsite("es_sitename", "es_carriertype", "news");
}
@Scheduled(cron = "0 15 1/3 * * *")
public void checkWechat() {
checkSocial("es_authors", "es_carriertype", "wechat", "5");
}
@Scheduled(cron = "0 0 2/4 * * *")
public void checkArticleSite() {
checkWebsite("es_sitename", "es_carriertype", "article");
}
public void checkWebsite(String aggFieldName, String queryFieldName, String queryFieldValue) {
try {
Map<String, SearchAggregation> searchAggregationMap = performAggregationSearch(
esIp, esPort, esUsername, esPassword, aggFieldName, queryFieldName, queryFieldValue);
JSONObject dataObject = new JSONObject();
dataObject.put("carrierType", queryFieldValue);
String rsp = HttpUtils.post(websiteQueryAPI, dataObject);
JSONObject rspObj = JSON.parseObject(rsp);
if (rspObj.getIntValue("code") == 200) {
JSONArray rspArr = rspObj.getJSONArray("content");
for (Object obj : rspArr) {
TargetWebsite targetWebsite = JSONObject.parseObject(obj.toString(), TargetWebsite.class);
String siteName = targetWebsite.getSiteName();
if (searchAggregationMap.containsKey(siteName)) {
SearchAggregation checkInfo = searchAggregationMap.get(siteName);
targetWebsite.setCheckTotalNum(checkInfo.getCount());
targetWebsite.setCheckLastTime(checkInfo.getLastTime());
targetWebsite.setCheckUpdateTime(new Date());
String updateRsp = HttpUtils.post(websiteUpdateAPI, targetWebsite);
JSONObject updateRspObj = JSONObject.parseObject(updateRsp);
if (updateRspObj.getIntValue("code") != 200) {
logger.warning("更新站点【" + siteName + "】巡检信息失败");
}
Thread.sleep(updateInterval);
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
logger.info("站点巡检完毕");
}
public void checkSocial(String aggFieldName, String queryFieldName, String queryFieldValue, String socialTypeCode) {
try {
Map<String, SearchAggregation> searchAggregationMap = performAggregationSearch(
esIp, esPort, esUsername, esPassword, aggFieldName, queryFieldName, queryFieldValue);
TargetSocial postData = new TargetSocial();
postData.setUserFlag("0");
postData.setUserType(socialTypeCode);
String rsp = HttpUtils.post(socialQueryAPI, postData);
JSONObject rspObj = JSON.parseObject(rsp);
if (rspObj.getIntValue("code") == 200) {
JSONArray rspArr = rspObj.getJSONArray("content");
for (Object obj : rspArr) {
TargetSocial targetSocial = JSONObject.parseObject(obj.toString(), TargetSocial.class);
String userName = targetSocial.getUserName();
if (searchAggregationMap.containsKey(userName)) {
SearchAggregation checkInfo = searchAggregationMap.get(userName);
targetSocial.setCheckTotalNum(checkInfo.getCount());
targetSocial.setCheckLastTime(checkInfo.getLastTime());
targetSocial.setCheckUpdateTime(new Date());
String updateRsp = HttpUtils.post(socialUpdateAPI, targetSocial);
JSONObject updateRspObj = JSONObject.parseObject(updateRsp);
if (updateRspObj.getIntValue("code") != 200) {
logger.warning("更新账号【" + userName + "】巡检信息失败");
}
Thread.sleep(updateInterval);
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
logger.info("社交帐号巡检完毕");
}
// @Scheduled(cron = "0 45 0/3 * * *")
// public void checkNewsSite() {
// checkWebsite("es_sitename", "es_carriertype", "news");
// }
//
// @Scheduled(cron = "0 15 1/3 * * *")
// public void checkWechat() {
// checkSocial("es_authors", "es_carriertype", "wechat", "5");
// }
//
// @Scheduled(cron = "0 0 2/4 * * *")
// public void checkArticleSite() {
// checkWebsite("es_sitename", "es_carriertype", "article");
// }
}