import cn.reghao.dfs.store.DfsStoreApplication; import cn.reghao.dfs.store.db.mapper.FileMetaMapper; import cn.reghao.dfs.store.model.po.FileMeta; import cn.reghao.jutil.jdk.db.Page; import cn.reghao.jutil.jdk.security.DigestUtil; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit4.SpringRunner; import java.io.File; import java.io.IOException; import java.nio.file.FileVisitResult; import java.nio.file.FileVisitor; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; import java.security.NoSuchAlgorithmException; import java.util.List; /** * @author reghao * @date 2023-03-22 16:19:12 */ @Slf4j @ActiveProfiles("dev") @SpringBootTest(classes = DfsStoreApplication.class) @RunWith(SpringRunner.class) public class FileMetaTest { @Autowired FileMetaMapper fileMetaMapper; // 正则表达式查询测试 public void regexpTest() { String bucket = ""; String prefix = "abc/d/"; String startAfter = "abc/d/e"; Integer maxKeys = 10; StringBuilder regex = new StringBuilder(); regex.append("^").append(prefix).append("([^/])+/?$"); if (startAfter.isBlank()) { fileMetaMapper.findAll0(bucket, maxKeys, regex.toString()); } else { fileMetaMapper.findAll2(bucket, prefix, startAfter, maxKeys); } } /** * 分页处理 * * @param * @return * @date 2023-05-30 00:26:07 */ @Test public void processByPage() { int pageSize = 10_000; int pageNumber = 1; Page page = new Page(pageNumber, pageSize); List list = fileMetaMapper.findFileMetaByPage(page); while (!list.isEmpty()) { list.forEach(fileMeta -> { }); pageNumber++; page = new Page(pageNumber, pageSize); list = fileMetaMapper.findFileMetaByPage(page); log.info("page -> {}", pageNumber); } } public void walkDir(Path path) throws IOException { Files.walkFileTree(path, new FileVisitor<>() { @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { String absolutePath = file.toString(); process(absolutePath); return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { return FileVisitResult.CONTINUE; } @Override public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { return FileVisitResult.CONTINUE; } }); } private void process(String absolutePath) { try { String sha256sum = DigestUtil.sha256sum(absolutePath); FileMeta fileMeta = fileMetaMapper.findBySha256sum(sha256sum); if (fileMeta != null) { FileUtils.deleteQuietly(new File(absolutePath)); } else { log.info("{} 在 db 中不存在", absolutePath); } } catch (IOException | NoSuchAlgorithmException e) { e.printStackTrace(); } } @Test public void deleteFile() throws IOException { String baseDir = "/home/reghao/mnt/porn/0.已完成"; baseDir = "/run/media/reghao/7c43343d-c1e2-4e68-bc18-89bd3f61b0c8/spider/porn/"; Path path = Path.of(baseDir); walkDir(path); } }