diff --git a/dsp/src/main/java/com/jsc/dsp/utils/FileUtils.java b/dsp/src/main/java/com/jsc/dsp/utils/FileUtils.java index 792a35d..694405a 100644 --- a/dsp/src/main/java/com/jsc/dsp/utils/FileUtils.java +++ b/dsp/src/main/java/com/jsc/dsp/utils/FileUtils.java @@ -1,23 +1,25 @@ package com.jsc.dsp.utils; import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import java.io.*; import java.net.HttpURLConnection; import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.util.Calendar; -import java.util.Date; -import java.util.HashSet; -import java.util.logging.Logger; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class FileUtils { - private final Logger logger = Logger.getLogger(this.getClass().getName()); + private final Logger logger = LoggerFactory.getLogger(this.getClass().getName()); public FileUtils() { } @@ -79,7 +81,7 @@ public class FileUtils { public int downloadFromUrl(String urlStr, String savePath) { try { if (downloadedFileSet.contains(urlStr)) { - logger.warning("File exist from " + urlStr); + logger.warn("File exist from " + urlStr); return 2; } String[] urlCascade = urlStr.split("/"); @@ -183,6 +185,180 @@ public class FileUtils { } } + + /** + * 解压tar.gz文件到指定目录 + */ + /** + * 解压tar.gz文件到指定目录，返回解压的文件数量（不包含目录） + * + * @return 解压的普通文件数量 + */ + public static int extractTarGz(File tarFile, File destDir) throws IOException { + if (!destDir.exists() && !destDir.mkdirs()) { + throw new IOException("无法创建目标目录: " + destDir.getAbsolutePath()); + } + + int fileCount = 0; + + try (FileInputStream fis = new FileInputStream(tarFile); + BufferedInputStream bis = new BufferedInputStream(fis); + GzipCompressorInputStream gzIn = new GzipCompressorInputStream(bis); + TarArchiveInputStream tarIn = new TarArchiveInputStream(gzIn)) { + + TarArchiveEntry entry; + + while ((entry = tarIn.getNextTarEntry()) != null) { + // 跳过空条目、符号链接、特殊设备文件 + if (entry.getName().trim().isEmpty() + || entry.isSymbolicLink() + || entry.isCharacterDevice() + || entry.isBlockDevice()) { + continue; + } + + // 安全校验：防止路径遍历攻击 + Path entryPath = destDir.toPath().resolve(entry.getName()).normalize(); + if (!entryPath.startsWith(destDir.toPath().normalize())) { + continue; + } + + // 创建目录结构（为后续文件写入做准备） + if (entry.isDirectory()) { + Files.createDirectories(entryPath); + } else { + Files.createDirectories(entryPath.getParent()); + Files.copy(tarIn, entryPath, StandardCopyOption.REPLACE_EXISTING); + fileCount++; + } + } + return fileCount; + + } catch (IOException e) { + throw e; + } + } + + /** + * 递归删除目录（含子目录和文件） + */ + public static void deleteDirectory(Path path) throws IOException { + if (!Files.exists(path)) return; + + Files.walkFileTree(path, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } + + public static void moveAllFilesRecursively(Path sourceDir, Path targetDir) throws IOException { + if (!Files.exists(sourceDir) || !Files.isDirectory(sourceDir)) { + return; + } + + // 使用Files.walk递归遍历所有文件 + try (Stream walkStream = Files.walk(sourceDir)) { + walkStream + .filter(path -> !Files.isDirectory(path)) // 只处理文件 + .sorted() // 确保先创建父目录再移动文件 + .forEach(file -> { + try { + // 计算相对路径（相对于sourceDir） + Path relativePath = sourceDir.relativize(file); + + // 构建目标文件路径 + Path targetFile = targetDir.resolve(relativePath); + + // 确保目标父目录存在 + Files.createDirectories(targetFile.getParent()); + + // 移动文件（覆盖同名文件） + Files.move(file, targetFile, + StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.COPY_ATTRIBUTES); + + } catch (IOException e) { + throw new UncheckedIOException(e); // 便于Stream中抛出 + } + }); + } catch (UncheckedIOException e) { + throw e.getCause() instanceof IOException ? (IOException) e.getCause() : new IOException(e); + } + } + + /** + * 递归遍历源目录，将所有文件平铺移动到目标目录（不保留目录结构，同名覆盖） + * + * @param sourceDir 源目录（临时解压目录） + * @param targetDir 目标目录（如 D:/output/twitter/videos） + * @return 成功移动的文件数量 + */ + public static int flattenAndMoveFiles(Path sourceDir, Path targetDir) throws Exception { + if (!Files.exists(sourceDir) || !Files.isDirectory(sourceDir)) { + return 0; + } + + AtomicInteger movedCount = new AtomicInteger(0); + Map duplicateFiles = new HashMap<>(); // 记录被覆盖的文件 + + try (Stream walkStream = Files.walk(sourceDir)) { + walkStream + .filter(path -> Files.isRegularFile(path)) // 只处理普通文件 + .forEach(file -> { + try { + String fileName = file.getFileName().toString(); + Path targetFile = targetDir.resolve(fileName); + + // 检测同名文件覆盖（用于日志记录） + boolean willOverwrite = Files.exists(targetFile); + if (willOverwrite) { + duplicateFiles.put(fileName, file); + } + + // 移动文件（覆盖同名文件） + Files.move(file, targetFile); + + movedCount.incrementAndGet(); + + } catch (Exception e) { + e.printStackTrace(); + } + }); + } catch (UncheckedIOException e) { + throw e.getCause() instanceof IOException ? (IOException) e.getCause() : new IOException(e); + } + + return movedCount.get(); + } + + + /** + * 清空目录内容（保留目录本身） + */ + public static void cleanDirectory(Path dir) throws IOException { + if (!Files.exists(dir)) return; + + try (DirectoryStream stream = Files.newDirectoryStream(dir)) { + for (Path entry : stream) { + if (Files.isDirectory(entry)) { + deleteDirectory(entry); + } else { + Files.delete(entry); + } + } + } + } + + public static void main(String[] args) { saveStringToFile("{\"aaa\":\"测试测试testtest\"}", "E:/yuxin/test.json"); }