import cn.reghao.dfs.store.DfsStoreApplication; import cn.reghao.dfs.store.db.mapper.*; import cn.reghao.dfs.store.model.po.*; import cn.reghao.dfs.store.service.ObjectNameService; import cn.reghao.dfs.store.util.UserContext; import cn.reghao.jutil.jdk.db.Page; import cn.reghao.jutil.jdk.security.DigestUtil; import cn.reghao.jutil.jdk.thread.ThreadPoolWrapper; import cn.reghao.oss.api.constant.ObjectScope; import cn.reghao.oss.api.constant.UploadChannel; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit4.SpringRunner; import java.io.File; import java.io.IOException; import java.nio.file.FileVisitResult; import java.nio.file.FileVisitor; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; import java.security.NoSuchAlgorithmException; import java.util.List; import java.util.concurrent.ExecutorService; /** * @author reghao * @date 2023-07-16 19:19:12 */ @Slf4j @ActiveProfiles("dev") @SpringBootTest(classes = DfsStoreApplication.class) @RunWith(SpringRunner.class) public class FileMetaTest { @Autowired FileMetaMapper fileMetaMapper; // 正则表达式查询测试 public void regexpTest() { String bucket = ""; String prefix = "abc/d/"; String startAfter = "abc/d/e"; Integer maxKeys = 10; StringBuilder regex = new StringBuilder(); regex.append("^").append(prefix).append("([^/])+/?$"); if (startAfter.isBlank()) { fileMetaMapper.findAll0(bucket, maxKeys, regex.toString()); } else { fileMetaMapper.findAll2(bucket, prefix, startAfter, maxKeys); } } /** * 分页处理 * * @param * @return * @date 2023-05-30 00:26:07 */ @Test public void processByPage() { int pageSize = 10_000; int pageNumber = 1; Page page = new Page(pageNumber, pageSize); List list = fileMetaMapper.findFileMetaByPage(page); while (!list.isEmpty()) { list.forEach(fileMeta -> { }); pageNumber++; page = new Page(pageNumber, pageSize); list = fileMetaMapper.findFileMetaByPage(page); log.info("page -> {}", pageNumber); } } public void walkDir(Path path) throws IOException { Files.walkFileTree(path, new FileVisitor<>() { @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { String absolutePath = file.toString(); process(absolutePath); return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { return FileVisitResult.CONTINUE; } @Override public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { return FileVisitResult.CONTINUE; } }); } private void process(String absolutePath) { try { String sha256sum = DigestUtil.sha256sum(absolutePath); FileMeta fileMeta = fileMetaMapper.findBySha256sum(sha256sum); if (fileMeta != null) { FileUtils.deleteQuietly(new File(absolutePath)); log.info("{} 删除 {} -> {}", Thread.currentThread().getName(), total++, absolutePath); } else { //log.info("{} 在 db 中不存在", absolutePath); } } catch (IOException | NoSuchAlgorithmException e) { e.printStackTrace(); } } ExecutorService threadPool = ThreadPoolWrapper.threadPool("delete-pool", 10); int total = 1; @Test public void deleteFile() throws IOException, InterruptedException { String baseDir = "/home/reghao/mnt1/spider/bili/img/vidcover"; //baseDir = "/run/media/reghao/7c43343d-c1e2-4e68-bc18-89bd3f61b0c8/cover"; //baseDir = "/home/reghao/mnt1/spider/bili/img/avatar"; Path path = Path.of(baseDir); //walkDir(path); threadPool.submit(new Task("/home/reghao/mnt1/spider/bili/img/avatar")); threadPool.submit(new Task("/run/media/reghao/7c43343d-c1e2-4e68-bc18-89bd3f61b0c8/cover")); threadPool.submit(new Task("/run/media/reghao/7c43343d-c1e2-4e68-bc18-89bd3f61b0c8/avatar")); Thread.sleep(3600_000*24*7); } class Task implements Runnable { private final String baseDir; public Task(String baseDir) { this.baseDir = baseDir; } @Override public void run() { Path path = Path.of(baseDir); try { walkDir(path); } catch (IOException e) { e.printStackTrace(); } } } @Autowired ObjectNameService objectNameService; @Test public void initUploadChannel() { UserContext userContext = new UserContext(10001L); for (UploadChannel channel : UploadChannel.values()) { String objectName = channel.getPrefix(); objectNameService.createParentDirs(objectName, ObjectScope.PUBLIC.getCode()); } } @Autowired DataBlockMapper dataBlockMapper; @Test public void dataBlockTest() { int pageSize = 10000; int nextId = 0; List list = dataBlockMapper.findDataBlocks(pageSize, nextId); while (!list.isEmpty()) { process(list); nextId = list.get(list.size()-1).getId(); list = dataBlockMapper.findDataBlocks(pageSize, nextId); log.info("nextId -> {}", nextId); } } private void process(List list) { for (DataBlock dataBlock : list) { String contentId = dataBlock.getContentId(); List list1 = fileMetaMapper.findByContentId(contentId); if (list1.isEmpty()) { log.info("{} not exist in file_meta", contentId); } } } }