Compare commits
3 Commits
bceee18244
...
5cc725fae9
| Author | SHA1 | Date | |
|---|---|---|---|
|
5cc725fae9 | ||
|
|
f62c55de99 | ||
|
|
4d2035b9fa |
231
dsp/dsp.iml
231
dsp/dsp.iml
@ -1,240 +1,13 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
|
<module version="4">
|
||||||
<component name="FacetManager">
|
<component name="FacetManager">
|
||||||
<facet type="web" name="Web">
|
|
||||||
<configuration>
|
|
||||||
<webroots />
|
|
||||||
</configuration>
|
|
||||||
</facet>
|
|
||||||
<facet type="Spring" name="Spring">
|
|
||||||
<configuration />
|
|
||||||
</facet>
|
|
||||||
<facet type="jpa" name="JPA">
|
<facet type="jpa" name="JPA">
|
||||||
<configuration>
|
<configuration>
|
||||||
<setting name="validation-enabled" value="true" />
|
<setting name="validation-enabled" value="true" />
|
||||||
<setting name="provider-name" value="Hibernate" />
|
<setting name="provider-name" value="Hibernate" />
|
||||||
<datasource-mapping>
|
<datasource-mapping />
|
||||||
<factory-entry name="entityManagerFactory" />
|
|
||||||
</datasource-mapping>
|
|
||||||
<naming-strategy-map />
|
<naming-strategy-map />
|
||||||
</configuration>
|
</configuration>
|
||||||
</facet>
|
</facet>
|
||||||
</component>
|
</component>
|
||||||
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
|
|
||||||
<output url="file://$MODULE_DIR$/target/classes" />
|
|
||||||
<output-test url="file://$MODULE_DIR$/target/test-classes" />
|
|
||||||
<content url="file://$MODULE_DIR$">
|
|
||||||
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
|
|
||||||
<sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
|
|
||||||
<excludeFolder url="file://$MODULE_DIR$/target" />
|
|
||||||
</content>
|
|
||||||
<orderEntry type="inheritedJdk" />
|
|
||||||
<orderEntry type="sourceFolder" forTests="false" />
|
|
||||||
<orderEntry type="library" name="Maven: org.jetbrains:annotations:26.0.2-1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.commons:commons-compress:1.18" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.alibaba:fastjson:1.2.75" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.projectlombok:lombok:1.18.16" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-web:2.4.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter:2.4.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-logging:2.4.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: ch.qos.logback:logback-classic:1.2.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: ch.qos.logback:logback-core:1.2.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-to-slf4j:2.13.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.slf4j:jul-to-slf4j:1.7.30" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: jakarta.annotation:jakarta.annotation-api:1.3.5" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.yaml:snakeyaml:1.27" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-json:2.4.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.11.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.11.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.fasterxml.jackson.module:jackson-module-parameter-names:2.11.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-tomcat:2.4.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.tomcat.embed:tomcat-embed-core:9.0.41" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.glassfish:jakarta.el:3.0.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.tomcat.embed:tomcat-embed-websocket:9.0.41" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework:spring-web:5.3.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework:spring-beans:5.3.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework:spring-webmvc:5.3.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework:spring-aop:5.3.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework:spring-expression:5.3.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-data-elasticsearch:2.4.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.data:spring-data-elasticsearch:4.1.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework:spring-tx:5.3.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.data:spring-data-commons:2.4.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.elasticsearch.plugin:transport-netty4-client:7.7.0" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: io.netty:netty-buffer:4.1.55.Final" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: io.netty:netty-codec:4.1.55.Final" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: io.netty:netty-codec-http:4.1.55.Final" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: io.netty:netty-common:4.1.55.Final" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: io.netty:netty-handler:4.1.55.Final" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: io.netty:netty-resolver:4.1.55.Final" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: io.netty:netty-transport:4.1.55.Final" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.elasticsearch.client:elasticsearch-rest-high-level-client:7.7.0" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.elasticsearch:elasticsearch:7.7.0" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.elasticsearch:elasticsearch-core:7.7.0" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.elasticsearch:elasticsearch-secure-sm:7.7.0" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.elasticsearch:elasticsearch-x-content:7.7.0" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.11.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.11.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.11.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.elasticsearch:elasticsearch-geo:7.7.0" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-core:8.5.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-analyzers-common:8.5.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-backward-codecs:8.5.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-grouping:8.5.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-highlighter:8.5.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-join:8.5.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-memory:8.5.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-misc:8.5.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-queries:8.5.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-queryparser:8.5.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-sandbox:8.5.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-spatial-extras:8.5.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-spatial3d:8.5.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-suggest:8.5.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.elasticsearch:elasticsearch-cli:7.7.0" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.carrotsearch:hppc:0.8.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: joda-time:joda-time:2.10.4" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.tdunning:t-digest:3.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.hdrhistogram:HdrHistogram:2.1.9" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.elasticsearch:jna:4.5.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.elasticsearch.client:elasticsearch-rest-client:7.7.0" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.5.13" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.4.14" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpasyncclient:4.1.4" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore-nio:4.4.14" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.elasticsearch.plugin:mapper-extras-client:7.7.0" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.elasticsearch.plugin:parent-join-client:7.7.0" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.elasticsearch.plugin:aggs-matrix-stats-client:7.7.0" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.elasticsearch.plugin:rank-eval-client:7.7.0" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.elasticsearch.plugin:lang-mustache-client:7.7.0" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.github.spullara.mustache.java:compiler:0.9.6" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.11.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.30" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.cloud:spring-cloud-stream:3.0.7.RELEASE" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-validation:2.4.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.hibernate.validator:hibernate-validator:6.1.6.Final" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: jakarta.validation:jakarta.validation-api:2.0.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework:spring-messaging:5.3.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.integration:spring-integration-core:5.4.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: io.projectreactor:reactor-core:3.4.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.reactivestreams:reactive-streams:1.0.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.integration:spring-integration-jmx:5.4.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.retry:spring-retry:1.3.0" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: javax.annotation:javax.annotation-api:1.3.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.cloud:spring-cloud-function-context:3.0.9.RELEASE" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: net.jodah:typetools:0.6.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.cloud:spring-cloud-function-core:3.0.9.RELEASE" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.cloud:spring-cloud-stream-binder-kafka:3.0.7.RELEASE" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.cloud:spring-cloud-stream-binder-kafka-core:3.0.7.RELEASE" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.integration:spring-integration-kafka:5.4.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.kafka:kafka-clients:2.6.0" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.github.luben:zstd-jni:1.4.4-7" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.lz4:lz4-java:1.7.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.xerial.snappy:snappy-java:1.1.7.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.kafka:spring-kafka:2.6.4" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.google.protobuf:protobuf-java:3.11.4" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.squareup.okhttp3:okhttp:4.8.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.squareup.okio:okio:2.7.0" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.jetbrains.kotlin:kotlin-stdlib-common:1.4.21" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.jetbrains.kotlin:kotlin-stdlib:1.4.21" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.springframework.boot:spring-boot-starter-test:2.4.1" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.springframework.boot:spring-boot-test:2.4.1" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.springframework.boot:spring-boot-test-autoconfigure:2.4.1" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: com.jayway.jsonpath:json-path:2.4.0" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: net.minidev:json-smart:2.3" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: net.minidev:accessors-smart:1.2" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.ow2.asm:asm:5.0.4" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: jakarta.xml.bind:jakarta.xml.bind-api:2.3.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: jakarta.activation:jakarta.activation-api:1.2.2" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.assertj:assertj-core:3.18.1" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest:2.2" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.junit.jupiter:junit-jupiter:5.7.0" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.junit.jupiter:junit-jupiter-params:5.7.0" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.junit.jupiter:junit-jupiter-engine:5.7.0" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.junit.platform:junit-platform-engine:1.7.0" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.mockito:mockito-core:3.6.28" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: net.bytebuddy:byte-buddy:1.10.18" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: net.bytebuddy:byte-buddy-agent:1.10.18" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.objenesis:objenesis:3.1" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.mockito:mockito-junit-jupiter:3.6.28" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.skyscreamer:jsonassert:1.5.0" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: com.vaadin.external.google:android-json:0.0.20131108.vaadin1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework:spring-core:5.3.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework:spring-jcl:5.3.2" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.springframework:spring-test:5.3.2" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.xmlunit:xmlunit-core:2.7.0" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.springframework.cloud:spring-cloud-stream-test-support:3.0.7.RELEASE" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-autoconfigure:2.4.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.11.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.11.3" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.springframework.kafka:spring-kafka-test:2.6.4" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework:spring-context:5.3.2" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.apache.kafka:kafka-clients:test:2.6.0" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.apache.kafka:kafka-streams:2.6.0" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.apache.kafka:connect-json:2.6.0" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.apache.kafka:connect-api:2.6.0" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.rocksdb:rocksdbjni:5.18.4" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.apache.kafka:kafka-streams-test-utils:2.6.0" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.apache.kafka:kafka_2.13:2.6.0" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: com.fasterxml.jackson.module:jackson-module-scala_2.13:2.11.3" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: com.fasterxml.jackson.module:jackson-module-paranamer:2.11.3" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: com.thoughtworks.paranamer:paranamer:2.8" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.11.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: net.sf.jopt-simple:jopt-simple:5.0.4" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: com.yammer.metrics:metrics-core:2.2.0" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.scala-lang.modules:scala-collection-compat_2.13:2.1.6" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.scala-lang.modules:scala-java8-compat_2.13:0.9.1" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.scala-lang:scala-library:2.13.2" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.scala-lang:scala-reflect:2.13.2" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: com.typesafe.scala-logging:scala-logging_2.13:3.9.2" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.apache.zookeeper:zookeeper:3.5.8" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.apache.zookeeper:zookeeper-jute:3.5.8" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.apache.yetus:audience-annotations:0.5.0" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: io.netty:netty-transport-native-epoll:4.1.55.Final" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: io.netty:netty-transport-native-unix-common:4.1.55.Final" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: commons-cli:commons-cli:1.4" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.apache.kafka:kafka_2.13:test:2.6.0" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.junit.jupiter:junit-jupiter-api:5.7.0" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.apiguardian:apiguardian-api:1.1.0" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.opentest4j:opentest4j:1.2.0" level="project" />
|
|
||||||
<orderEntry type="library" scope="TEST" name="Maven: org.junit.platform:junit-platform-commons:1.7.0" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-devtools:2.4.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot:2.4.1" level="project" />
|
|
||||||
<orderEntry type="library" scope="RUNTIME" name="Maven: mysql:mysql-connector-java:8.0.22" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: commons-net:commons-net:3.10.0" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-data-jpa:2.4.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-aop:2.4.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.aspectj:aspectjweaver:1.9.6" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-jdbc:2.4.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.zaxxer:HikariCP:3.4.5" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework:spring-jdbc:5.3.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: jakarta.transaction:jakarta.transaction-api:1.3.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: jakarta.persistence:jakarta.persistence-api:2.2.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.hibernate:hibernate-core:5.4.25.Final" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.jboss.logging:jboss-logging:3.4.1.Final" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.javassist:javassist:3.27.0-GA" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: antlr:antlr:2.7.7" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.jboss:jandex:2.1.3.Final" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.fasterxml:classmate:1.5.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.dom4j:dom4j:2.1.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.hibernate.common:hibernate-commons-annotations:5.1.2.Final" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.glassfish.jaxb:jaxb-runtime:2.3.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.glassfish.jaxb:txw2:2.3.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.sun.istack:istack-commons-runtime:3.0.11" level="project" />
|
|
||||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.sun.activation:jakarta.activation:1.2.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework.data:spring-data-jpa:2.4.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework:spring-orm:5.3.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.springframework:spring-aspects:5.3.2" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.poi:poi:5.2.4" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.15" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.commons:commons-collections4:4.4" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.commons:commons-math3:3.6.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: commons-io:commons-io:2.13.0" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.zaxxer:SparseBitSet:1.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-api:2.13.3" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.poi:poi-ooxml:5.2.4" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.poi:poi-ooxml-lite:5.2.4" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: org.apache.xmlbeans:xmlbeans:5.1.1" level="project" />
|
|
||||||
<orderEntry type="library" name="Maven: com.github.virtuald:curvesapi:1.08" level="project" />
|
|
||||||
</component>
|
|
||||||
</module>
|
</module>
|
||||||
@ -44,4 +44,14 @@ public class ExportController {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/triggerTwitterTask")
|
||||||
|
public ReturnT<String> triggerTwitterTask() {
|
||||||
|
try {
|
||||||
|
new Thread(() -> autoExportAndUpload.exportTwitterDataAndUpload()).start();
|
||||||
|
return new ReturnT<>(200, "", "");
|
||||||
|
} catch (Exception e) {
|
||||||
|
return new ReturnT<>(500, e.getMessage(), "");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,12 @@
|
|||||||
|
package com.jsc.dsp.dao;
|
||||||
|
|
||||||
|
import com.jsc.dsp.model.EsDataTwitterView;
|
||||||
|
import org.springframework.data.jpa.repository.JpaRepository;
|
||||||
|
import org.springframework.stereotype.Repository;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Repository
|
||||||
|
public interface EsDataTwitterRepository extends JpaRepository<EsDataTwitterView, String> {
|
||||||
|
List<EsDataTwitterView> findAllByEsLoadtimeAfter(String loadtime);
|
||||||
|
}
|
||||||
54
dsp/src/main/java/com/jsc/dsp/model/EsDataTwitterView.java
Normal file
54
dsp/src/main/java/com/jsc/dsp/model/EsDataTwitterView.java
Normal file
@ -0,0 +1,54 @@
|
|||||||
|
package com.jsc.dsp.model;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import javax.persistence.Entity;
|
||||||
|
import javax.persistence.Id;
|
||||||
|
import javax.persistence.Table;
|
||||||
|
|
||||||
|
@Entity
|
||||||
|
@Data
|
||||||
|
@Table(name = "es_data_twitter")
|
||||||
|
public class EsDataTwitterView {
|
||||||
|
|
||||||
|
@Id
|
||||||
|
private String esUrltime;
|
||||||
|
|
||||||
|
private String esAuthors;
|
||||||
|
|
||||||
|
private String esCarriertype;
|
||||||
|
|
||||||
|
private String esSitename;
|
||||||
|
|
||||||
|
private String esUrlcontent;
|
||||||
|
|
||||||
|
private String esUrlcontentTranslate;
|
||||||
|
|
||||||
|
private String esUrlname;
|
||||||
|
|
||||||
|
private String esUrltitle;
|
||||||
|
|
||||||
|
private String esUrltitleTranslate;
|
||||||
|
|
||||||
|
private String esVideo;
|
||||||
|
|
||||||
|
private String esExtname;
|
||||||
|
|
||||||
|
private String esIsrepost;
|
||||||
|
|
||||||
|
private String esCatalog1;
|
||||||
|
|
||||||
|
private String esForwardcount;
|
||||||
|
|
||||||
|
private String esLikecount;
|
||||||
|
|
||||||
|
private String esCommentcount;
|
||||||
|
|
||||||
|
private String esHkey;
|
||||||
|
|
||||||
|
private String esUrlimage;
|
||||||
|
|
||||||
|
private String esUserid;
|
||||||
|
|
||||||
|
private String esLoadtime;
|
||||||
|
}
|
||||||
@ -48,6 +48,9 @@ public class StorageService extends StreamService {
|
|||||||
@Value("${custom.websiteWhiteList}")
|
@Value("${custom.websiteWhiteList}")
|
||||||
String websiteWhiteListString;
|
String websiteWhiteListString;
|
||||||
|
|
||||||
|
@Value("${custom.twitterWhiteList}")
|
||||||
|
String twitterWhiteListString;
|
||||||
|
|
||||||
@Resource
|
@Resource
|
||||||
DatabaseConnector databaseConnector;
|
DatabaseConnector databaseConnector;
|
||||||
|
|
||||||
@ -66,6 +69,8 @@ public class StorageService extends StreamService {
|
|||||||
@StreamListener(StorageBinding.STORAGE_PIPELINE_IN)
|
@StreamListener(StorageBinding.STORAGE_PIPELINE_IN)
|
||||||
public void receiveMessage(Object payload) {
|
public void receiveMessage(Object payload) {
|
||||||
List<String> websiteWhiteList = Arrays.asList(websiteWhiteListString.split(";"));
|
List<String> websiteWhiteList = Arrays.asList(websiteWhiteListString.split(";"));
|
||||||
|
List<String> twitterWhiteList = Arrays.asList(twitterWhiteListString.split(";"));
|
||||||
|
|
||||||
String tempString;
|
String tempString;
|
||||||
try {
|
try {
|
||||||
tempString = new String(base64.decode(payload.toString()), StandardCharsets.UTF_8);
|
tempString = new String(base64.decode(payload.toString()), StandardCharsets.UTF_8);
|
||||||
@ -111,7 +116,7 @@ public class StorageService extends StreamService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// 只导出目标站点的数据
|
// 只导出白名单站点的数据
|
||||||
if (websiteWhiteList.contains(indeximos.getEs_sitename())) {
|
if (websiteWhiteList.contains(indeximos.getEs_sitename())) {
|
||||||
logger.info("开始处理站点【" + indeximos.getEs_sitename() + "】的数据入库流程");
|
logger.info("开始处理站点【" + indeximos.getEs_sitename() + "】的数据入库流程");
|
||||||
String uuid = UUID.randomUUID().toString().replaceAll("-", "");
|
String uuid = UUID.randomUUID().toString().replaceAll("-", "");
|
||||||
@ -142,6 +147,34 @@ public class StorageService extends StreamService {
|
|||||||
}
|
}
|
||||||
dbStorageItems.add(indeximos);
|
dbStorageItems.add(indeximos);
|
||||||
}
|
}
|
||||||
|
if (indeximos.getEs_carriertype().equals("media") && twitterWhiteList.contains(indeximos.getEs_authors())) {
|
||||||
|
logger.info("开始处理推特用户【" + indeximos.getEs_authors() + "】的数据入库流程");
|
||||||
|
String uuid = UUID.randomUUID().toString().replaceAll("-", "");
|
||||||
|
String es_urlname = indeximos.getEs_urlname();
|
||||||
|
if (!es_urlname.isEmpty()) {
|
||||||
|
// 根据urlname生成固定的UUID，避免重复入库相同的文章
|
||||||
|
UUID _uuid = UUID.nameUUIDFromBytes(es_urlname.getBytes());
|
||||||
|
uuid = _uuid.toString().replaceAll("-", "");
|
||||||
|
}
|
||||||
|
indeximos.setEs_sid(uuid);
|
||||||
|
indeximos.setEs_loadtime(StringUtils.TimestampToStringDate(System.currentTimeMillis()));
|
||||||
|
builder.setEsSid(uuid);
|
||||||
|
for (Field f : indeximos.getClass().getDeclaredFields()) {
|
||||||
|
f.setAccessible(true);
|
||||||
|
//判断字段是否为空，并且对象属性中的基本都会转为对象类型来判断
|
||||||
|
if (f.get(indeximos) == null) {
|
||||||
|
String fieldType = databaseConnector.getFieldType(Indeximos.class, f.getName());
|
||||||
|
if (fieldType.contains("Float")) {
|
||||||
|
f.set(indeximos, 0.0f);
|
||||||
|
} else {
|
||||||
|
if (!dateFields.contains(f.getName())) {
|
||||||
|
f.set(indeximos, "");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dbStorageItems.add(indeximos);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (dbStorageItems.size() > 0) {
|
if (dbStorageItems.size() > 0) {
|
||||||
databaseConnector.insertIntoDB(dbStorageItems);
|
databaseConnector.insertIntoDB(dbStorageItems);
|
||||||
|
|||||||
@ -77,6 +77,24 @@ public class AutoExportAndUpload {
|
|||||||
zipAndUploadDirectory(excelOutputPath, zipFileFullName, remoteZipPath);
|
zipAndUploadDirectory(excelOutputPath, zipFileFullName, remoteZipPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 每周一、三、五的早上8点，执行导出数据的任务
|
||||||
|
*/
|
||||||
|
@Scheduled(cron = "${custom.exportTaskSchedule}")
|
||||||
|
public void exportTwitterDataAndUpload() {
|
||||||
|
logger.info("开始导出twitter excel和pdf数据...");
|
||||||
|
String twitterLastLoadTime = configService.getConfigValueByName("twitter_last_loadtime");
|
||||||
|
String currentLoadTime = StringUtils.DateToString(new Date());
|
||||||
|
String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd"));
|
||||||
|
databaseConnector.twitterToXlsx(twitterLastLoadTime);
|
||||||
|
copyPagesFiles(twitterLastLoadTime, currentLoadTime);
|
||||||
|
configService.setConfigValueByName("twitter_last_loadtime", currentLoadTime);
|
||||||
|
String zipFileName = "data_twitter-" + timestamp + "-001.zip";
|
||||||
|
String zipFileFullName = backupFilePath + File.separator + zipFileName;
|
||||||
|
String remoteZipPath = ftpUploadPath + "/" + zipFileName;
|
||||||
|
zipAndUploadDirectory(excelOutputPath, zipFileFullName, remoteZipPath);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 将指定目录打包成 ZIP 文件（保存到指定本地路径），并上传到 FTP 服务器
|
* 将指定目录打包成 ZIP 文件（保存到指定本地路径），并上传到 FTP 服务器
|
||||||
*
|
*
|
||||||
|
|||||||
@ -1,8 +1,11 @@
|
|||||||
package com.jsc.dsp.utils;
|
package com.jsc.dsp.utils;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.jsc.dsp.dao.EsDataNewsRepository;
|
import com.jsc.dsp.dao.EsDataNewsRepository;
|
||||||
|
import com.jsc.dsp.dao.EsDataTwitterRepository;
|
||||||
import com.jsc.dsp.dao.IndeximosRepository;
|
import com.jsc.dsp.dao.IndeximosRepository;
|
||||||
import com.jsc.dsp.model.EsDataNewsView;
|
import com.jsc.dsp.model.EsDataNewsView;
|
||||||
|
import com.jsc.dsp.model.EsDataTwitterView;
|
||||||
import com.jsc.dsp.model.Indeximos;
|
import com.jsc.dsp.model.Indeximos;
|
||||||
import org.apache.poi.ss.usermodel.*;
|
import org.apache.poi.ss.usermodel.*;
|
||||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||||
@ -23,6 +26,8 @@ import java.nio.file.Paths;
|
|||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
import java.time.format.DateTimeFormatter;
|
import java.time.format.DateTimeFormatter;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
public class DatabaseConnector {
|
public class DatabaseConnector {
|
||||||
@ -32,10 +37,14 @@ public class DatabaseConnector {
|
|||||||
|
|
||||||
@Resource
|
@Resource
|
||||||
EsDataNewsRepository esDataNewsRepository;
|
EsDataNewsRepository esDataNewsRepository;
|
||||||
|
@Resource
|
||||||
|
EsDataTwitterRepository esDataTwitterRepository;
|
||||||
|
|
||||||
@Value("${custom.excelOutputPath}")
|
@Value("${custom.excelOutputPath}")
|
||||||
String excelOutputPath;
|
String excelOutputPath;
|
||||||
|
|
||||||
|
private static final ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
|
||||||
private final Logger logger = LoggerFactory.getLogger(this.getClass().getName());
|
private final Logger logger = LoggerFactory.getLogger(this.getClass().getName());
|
||||||
|
|
||||||
public void insertIntoDB(List<Indeximos> objectList) {
|
public void insertIntoDB(List<Indeximos> objectList) {
|
||||||
@ -78,7 +87,7 @@ public class DatabaseConnector {
|
|||||||
|
|
||||||
List<EsDataNewsView> esDataNewsViewList = esDataNewsRepository.findAllByEsLoadtimeAfter(startTime);
|
List<EsDataNewsView> esDataNewsViewList = esDataNewsRepository.findAllByEsLoadtimeAfter(startTime);
|
||||||
if (!esDataNewsViewList.isEmpty()) {
|
if (!esDataNewsViewList.isEmpty()) {
|
||||||
Field[] fields = esDataNewsViewList.get(0).getClass().getDeclaredFields();
|
Field[] fields = esDataNewsViewList.get(0).getClass().getDeclaredFields(); // 通过反射获取类的成员信息，并使用这些类成员为后续生成的excel表头做准备
|
||||||
try (Workbook workbook = new XSSFWorkbook();
|
try (Workbook workbook = new XSSFWorkbook();
|
||||||
ByteArrayOutputStream out = new ByteArrayOutputStream()) {
|
ByteArrayOutputStream out = new ByteArrayOutputStream()) {
|
||||||
|
|
||||||
@ -86,7 +95,7 @@ public class DatabaseConnector {
|
|||||||
|
|
||||||
// 创建表头
|
// 创建表头
|
||||||
Row headerRow = sheet.createRow(0);
|
Row headerRow = sheet.createRow(0);
|
||||||
CellStyle headerStyle = workbook.createCellStyle();
|
CellStyle headerStyle = workbook.createCellStyle(); // 创建单元格
|
||||||
headerStyle.setFillForegroundColor(IndexedColors.LIGHT_BLUE.getIndex());
|
headerStyle.setFillForegroundColor(IndexedColors.LIGHT_BLUE.getIndex());
|
||||||
headerStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND);
|
headerStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND);
|
||||||
|
|
||||||
@ -157,9 +166,186 @@ public class DatabaseConnector {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public void twitterToXlsx(String startTime) {
|
||||||
|
try {
|
||||||
|
Path dirPath = Paths.get(excelOutputPath);
|
||||||
|
if (!Files.exists(dirPath)) {
|
||||||
|
Files.createDirectories(dirPath);
|
||||||
|
}
|
||||||
|
String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd"));
|
||||||
|
String fileName = "data_twitter-" + timestamp + "-001.xlsx";
|
||||||
|
Path filePath = dirPath.resolve(fileName);
|
||||||
|
|
||||||
|
List<EsDataTwitterView> esDataNewsViewList = esDataTwitterRepository.findAllByEsLoadtimeAfter(startTime);
|
||||||
|
if (!esDataNewsViewList.isEmpty()) {
|
||||||
|
Field[] fields = esDataNewsViewList.get(0).getClass().getDeclaredFields(); // 通过反射获取类的成员信息，并使用这些类成员为后续生成的excel表头做准备
|
||||||
|
try (Workbook workbook = new XSSFWorkbook();
|
||||||
|
ByteArrayOutputStream out = new ByteArrayOutputStream()) {
|
||||||
|
|
||||||
|
Sheet sheet = workbook.createSheet("data");
|
||||||
|
|
||||||
|
// 创建表头
|
||||||
|
Row headerRow = sheet.createRow(0);
|
||||||
|
CellStyle headerStyle = workbook.createCellStyle(); // 创建单元格
|
||||||
|
headerStyle.setFillForegroundColor(IndexedColors.LIGHT_BLUE.getIndex());
|
||||||
|
headerStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND);
|
||||||
|
|
||||||
|
for (int i = 0; i < fields.length; i++) {
|
||||||
|
Cell cell = headerRow.createCell(i);
|
||||||
|
String formField = formField(fields[i]);
|
||||||
|
cell.setCellValue(formField);
|
||||||
|
cell.setCellStyle(headerStyle);
|
||||||
|
}
|
||||||
|
// 填充数据
|
||||||
|
int rowNum = 1;
|
||||||
|
for (EsDataTwitterView item : esDataNewsViewList) {
|
||||||
|
Row row = sheet.createRow(rowNum++);
|
||||||
|
logger.debug("导出excel第" + rowNum + "行");
|
||||||
|
// 0: esUrltime
|
||||||
|
row.createCell(0).setCellValue(item.getEsUrltime() != null ? item.getEsUrltime() : "");
|
||||||
|
|
||||||
|
// 1: esAuthors
|
||||||
|
row.createCell(1).setCellValue(item.getEsAuthors() != null ? item.getEsAuthors() : "");
|
||||||
|
|
||||||
|
// 2: esCarriertype
|
||||||
|
row.createCell(2).setCellValue(item.getEsCarriertype() != null ? item.getEsCarriertype() : "");
|
||||||
|
|
||||||
|
// 3: esSitename
|
||||||
|
row.createCell(3).setCellValue(item.getEsSitename() != null ? item.getEsSitename() : "");
|
||||||
|
|
||||||
|
// 4: esUrlcontent
|
||||||
|
String esUrlcontent = item.getEsUrlcontent();
|
||||||
|
if (esUrlcontent != null && esUrlcontent.length() > 10000) {
|
||||||
|
row.createCell(4).setCellValue(esUrlcontent.substring(0, 10000));
|
||||||
|
} else {
|
||||||
|
row.createCell(4).setCellValue(esUrlcontent != null ? esUrlcontent : "");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5: esUrlcontentTranslate
|
||||||
|
String esUrlcontentTranslate = item.getEsUrlcontentTranslate();
|
||||||
|
if (esUrlcontentTranslate != null && esUrlcontentTranslate.length() > 10000) {
|
||||||
|
row.createCell(5).setCellValue(esUrlcontentTranslate.substring(0, 10000));
|
||||||
|
} else {
|
||||||
|
row.createCell(5).setCellValue(esUrlcontentTranslate != null ? esUrlcontentTranslate : "");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 6: esUrlname
|
||||||
|
row.createCell(6).setCellValue(item.getEsUrlname() != null ? item.getEsUrlname() : "");
|
||||||
|
|
||||||
|
// 7: esUrltitle
|
||||||
|
String esUrltitle = item.getEsUrltitle();
|
||||||
|
if (esUrltitle != null && esUrltitle.length() > 10000) {
|
||||||
|
row.createCell(7).setCellValue(esUrltitle.substring(0, 10000));
|
||||||
|
} else {
|
||||||
|
row.createCell(7).setCellValue(esUrltitle != null ? esUrltitle : "");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 8: es_urltitle_translate
|
||||||
|
String esUrltitleTranslate = item.getEsUrltitleTranslate();
|
||||||
|
if (esUrltitleTranslate != null && esUrltitleTranslate.length() > 10000) {
|
||||||
|
row.createCell(7).setCellValue(esUrltitleTranslate.substring(0, 10000));
|
||||||
|
} else {
|
||||||
|
row.createCell(7).setCellValue(esUrltitleTranslate != null ? esUrltitleTranslate : "");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 9: esVideo
|
||||||
|
String videoFilenames = extractFilenamesFromJsonArray(item.getEsVideo());
|
||||||
|
row.createCell(9).setCellValue(videoFilenames);
|
||||||
|
|
||||||
|
// 10: esExtname
|
||||||
|
row.createCell(10).setCellValue(item.getEsExtname() != null ? item.getEsExtname() : "");
|
||||||
|
|
||||||
|
// 11: esIsrepost
|
||||||
|
row.createCell(11).setCellValue(item.getEsIsrepost() != null ? item.getEsIsrepost() : "");
|
||||||
|
|
||||||
|
// 12: esCatalog1
|
||||||
|
row.createCell(12).setCellValue(item.getEsCatalog1() != null ? item.getEsCatalog1() : "");
|
||||||
|
|
||||||
|
// 13: esForwardcount
|
||||||
|
row.createCell(13).setCellValue(item.getEsForwardcount() != null ? item.getEsForwardcount() : "");
|
||||||
|
|
||||||
|
// 14: esLikecount
|
||||||
|
row.createCell(14).setCellValue(item.getEsLikecount() != null ? item.getEsLikecount() : "");
|
||||||
|
|
||||||
|
// 15: esCommentcount
|
||||||
|
row.createCell(15).setCellValue(item.getEsCommentcount() != null ? item.getEsCommentcount() : "");
|
||||||
|
|
||||||
|
// 16: esHkey
|
||||||
|
row.createCell(16).setCellValue(item.getEsHkey() != null ? item.getEsHkey() : "");
|
||||||
|
|
||||||
|
// 17: esUrlimage
|
||||||
|
String imageFilenames = extractFilenamesFromJsonArray(item.getEsUrlimage());
|
||||||
|
row.createCell(17).setCellValue(imageFilenames);
|
||||||
|
|
||||||
|
// 18: esUserid
|
||||||
|
row.createCell(18).setCellValue(item.getEsUserid() != null ? item.getEsUserid() : "");
|
||||||
|
|
||||||
|
// 19: esLoadtime
|
||||||
|
row.createCell(19).setCellValue(item.getEsLoadtime() != null ? item.getEsLoadtime() : "");
|
||||||
|
|
||||||
|
}
|
||||||
|
logger.info("完成excel数据写入，共" + rowNum + "行");
|
||||||
|
|
||||||
|
// 自动调整列宽
|
||||||
|
for (int i = 0; i < fields.length; i++) {
|
||||||
|
sheet.autoSizeColumn(i);
|
||||||
|
}
|
||||||
|
|
||||||
|
workbook.write(out);
|
||||||
|
|
||||||
|
try (FileOutputStream fos = new FileOutputStream(filePath.toFile())) {
|
||||||
|
workbook.write(fos);
|
||||||
|
}
|
||||||
|
} catch (IOException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
logger.info("excel导出完成！");
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
private String formField(Field field) {
|
private String formField(Field field) {
|
||||||
String fieldString = field.getName();
|
String fieldString = field.getName();
|
||||||
return StringUtils.camelToSnake(fieldString);
|
return StringUtils.camelToSnake(fieldString);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
public String extractFilenamesFromJsonArray(String jsonStr) {
|
||||||
|
if (jsonStr == null || jsonStr.trim().isEmpty()) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
JsonNode array = objectMapper.readTree(jsonStr.replace("'", "\"").trim());
|
||||||
|
if (!array.isArray()) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
List<String> filenames = new ArrayList<>();
|
||||||
|
for (JsonNode node : array) {
|
||||||
|
if (node.has("path")) {
|
||||||
|
String url = node.get("path").asText().trim();
|
||||||
|
if (!url.isEmpty()) {
|
||||||
|
// 提取文件名（支持带参数的 URL）
|
||||||
|
String filename = url.split("\\?")[0]; // 去掉 ? 后的参数
|
||||||
|
filename = filename.substring(filename.lastIndexOf('/') + 1);
|
||||||
|
if (!filename.isEmpty()) {
|
||||||
|
filenames.add(filename);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return String.join(";", filenames);
|
||||||
|
} catch (Exception e) {
|
||||||
|
// 如果解析失败，返回空或原始内容（根据需求）
|
||||||
|
return ""; // 或者 return jsonStr; 用于调试
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -88,6 +88,7 @@ custom:
|
|||||||
socialQueryAPI: http://47.115.228.133:28081/api/open/target/social/queryAll?sortBy=id&shuffleResult=false
|
socialQueryAPI: http://47.115.228.133:28081/api/open/target/social/queryAll?sortBy=id&shuffleResult=false
|
||||||
socialUpdateAPI: http://47.115.228.133:28081/api/open/target/social/update
|
socialUpdateAPI: http://47.115.228.133:28081/api/open/target/social/update
|
||||||
websiteWhiteList: 能源界（国内信息）;能源界（国际信息）;中国能源新闻网;新华能源网;中国能源网（能源战略）;中国农网（三农要闻）;中国经济网（三农经济）;中华粮网（粮食安全）;美国之音（中国版面）;美国之音（中美关系）;美国之音（台海两岸版面）;美国之音（港澳版面）;看中国（看大陆版面）;看中国（重点新闻）;德国之声（中国报道）;纽约时报中文网（中国版面）;大纪元（一周大陆新闻）;EnergyNow;联合国粮农组织;路透社（中国版面）
|
websiteWhiteList: 能源界（国内信息）;能源界（国际信息）;中国能源新闻网;新华能源网;中国能源网（能源战略）;中国农网（三农要闻）;中国经济网（三农经济）;中华粮网（粮食安全）;美国之音（中国版面）;美国之音（中美关系）;美国之音（台海两岸版面）;美国之音（港澳版面）;看中国（看大陆版面）;看中国（重点新闻）;德国之声（中国报道）;纽约时报中文网（中国版面）;大纪元（一周大陆新闻）;EnergyNow;联合国粮农组织;路透社（中国版面）
|
||||||
|
twitterWhiteList: nytchinese;YesterdayBigcat;takaichi_sanae;yonhapcn;VOAChinese;ChineseWSJ;whyyoutouzhele;Jaemyung_Lee
|
||||||
excelOutputPath: D:/data/output/upload
|
excelOutputPath: D:/data/output/upload
|
||||||
backupFilePath: D:/data/output/backup
|
backupFilePath: D:/data/output/backup
|
||||||
pagesOutputPath: D:/data/output/pdf
|
pagesOutputPath: D:/data/output/pdf
|
||||||
|
|||||||
@ -192,9 +192,9 @@ class TwitterSpider(scrapy.Spider):
|
|||||||
item['es_catalog1'] = author_full_text
|
item['es_catalog1'] = author_full_text
|
||||||
# 判断是否需要翻译
|
# 判断是否需要翻译
|
||||||
if needs_translation(author_full_text):
|
if needs_translation(author_full_text):
|
||||||
item['es_content'] = translate_single(author_full_text) # TODO 翻译
|
item['es_catalog2'] = translate_single(author_full_text)
|
||||||
else:
|
else:
|
||||||
item['es_content'] = url_content
|
item['es_catalog2'] = ''
|
||||||
legacy = result['quoted_status_result']['result']['legacy']
|
legacy = result['quoted_status_result']['result']['legacy']
|
||||||
self.logger.info('采集引用推文原文信息')
|
self.logger.info('采集引用推文原文信息')
|
||||||
elif 'retweeted_status_result' in legacy:
|
elif 'retweeted_status_result' in legacy:
|
||||||
@ -208,7 +208,7 @@ class TwitterSpider(scrapy.Spider):
|
|||||||
if needs_translation(url_content):
|
if needs_translation(url_content):
|
||||||
item['es_content'] = translate_content_with_paragraphs(url_content) # TODO 翻译
|
item['es_content'] = translate_content_with_paragraphs(url_content) # TODO 翻译
|
||||||
else:
|
else:
|
||||||
item['es_content'] = url_content
|
item['es_content'] = ''
|
||||||
# 下载图片
|
# 下载图片
|
||||||
image_url_list = []
|
image_url_list = []
|
||||||
if 'entities' in legacy and 'media' in legacy['entities']:
|
if 'entities' in legacy and 'media' in legacy['entities']:
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user