import cn.reghao.dfs.store.DfsStoreApplication; import cn.reghao.dfs.store.db.mapper.DataBlockMapper; import cn.reghao.dfs.store.db.mapper.FileMetaMapper; import cn.reghao.dfs.store.model.po.DataBlock; import cn.reghao.dfs.store.model.po.FileMeta; import cn.reghao.dfs.store.model.vo.ObjectMeta; import cn.reghao.jutil.jdk.db.Page; import cn.reghao.jutil.jdk.security.DigestUtil; import lombok.extern.slf4j.Slf4j; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit4.SpringRunner; import java.io.FileInputStream; import java.io.IOException; import java.nio.file.FileVisitResult; import java.nio.file.FileVisitor; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.List; import java.util.UUID; /** * @author reghao * @date 2023-03-22 16:19:12 */ @Slf4j @ActiveProfiles("dev") @SpringBootTest(classes = DfsStoreApplication.class) @RunWith(SpringRunner.class) public class FileMetaTest { @Autowired FileMetaMapper fileMetaMapper; @Autowired DataBlockMapper dataBlockMapper; /** * 正则表达式查询测试 * * @param * @return * @date 2023-05-23 11:06:07 */ public void regexpTest() { String bucket = ""; String prefix = "abc/d/"; String startAfter = "abc/d/e"; Integer maxKeys = 10; StringBuilder regex = new StringBuilder(); regex.append("^").append(prefix).append("([^/])+/?$"); if (startAfter.isBlank()) { fileMetaMapper.findAll0(bucket, maxKeys, regex.toString()); } else { fileMetaMapper.findAll2(bucket, prefix, startAfter, maxKeys); } } @Test public void test2() { int pageSize = 1000; int pageNumber = 1; Page page = new Page(pageNumber, pageSize); List dataBlocks = dataBlockMapper.findDataBlockByPage(page); List fileMetas = new ArrayList<>(); while (!dataBlocks.isEmpty()) { dataBlocks.forEach(dataBlock -> { String objectId = dataBlock.getObjectId(); String contentId = UUID.randomUUID().toString().replace("-", ""); FileMeta fileMeta = fileMetaMapper.findByObjectId(objectId); if (fileMeta == null) { log.error("{} 不存在", objectId); return; } fileMeta.setContentId(contentId); fileMetas.add(fileMeta); dataBlock.setContentId(contentId); }); long start = System.currentTimeMillis(); if (!fileMetas.isEmpty()) { fileMetaMapper.updateBatch(fileMetas); fileMetas.clear(); log.info("batch update FileMeta cost {}", System.currentTimeMillis()-start); } if (!dataBlocks.isEmpty()) { dataBlockMapper.updateBatch(dataBlocks); log.info("batch update DataBlock cost {}", System.currentTimeMillis()-start); } pageNumber++; page = new Page(pageNumber, pageSize); dataBlocks = dataBlockMapper.findDataBlockByPage(page); log.info("page -> {}", pageNumber); } } public void walkDir(Path path) throws IOException { Files.walkFileTree(path, new FileVisitor<>() { @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { String absolutePath = file.toString(); try { FileInputStream fis = new FileInputStream(absolutePath); String sha256sum = DigestUtil.sha256sum(fis); FileMeta fileMeta = fileMetaMapper.findBySha256sum(sha256sum); if (fileMeta == null) { log.error("{} 不存在", absolutePath); } } catch (Exception ignore) { } return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { return FileVisitResult.CONTINUE; } @Override public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { return FileVisitResult.CONTINUE; } }); } @Test public void test22() throws IOException { String baseDir = "/home/reghao/mnt/zdata/porn/0.已完成/"; Path path = Path.of(baseDir); walkDir(path); } @Test public void test11() { int pageSize = 10_000; int pageNumber = 1; Page page = new Page(pageNumber, pageSize); List list = fileMetaMapper.findFileMetaByPage(page); while (!list.isEmpty()) { list.forEach(fileMeta -> { String objectName = fileMeta.getObjectName(); ObjectMeta objectMeta = fileMetaMapper.findObjectMeta(objectName); if (objectMeta == null) { log.error("{} 对应的文件不存在", objectName); } }); pageNumber++; page = new Page(pageNumber, pageSize); list = fileMetaMapper.findFileMetaByPage(page); log.info("page -> {}", pageNumber); } } }