oss/dfs-store/src/test/java/FileMetaTest.java

import cn.reghao.dfs.store.DfsStoreApplication;
import cn.reghao.dfs.store.db.mapper.DataBlockMapper;
import cn.reghao.dfs.store.db.mapper.FileMetaMapper;
import cn.reghao.dfs.store.model.po.DataBlock;
import cn.reghao.dfs.store.model.po.FileMeta;
import cn.reghao.dfs.store.model.vo.ObjectMeta;
import cn.reghao.jutil.jdk.db.Page;
import cn.reghao.jutil.jdk.security.DigestUtil;
import lombok.extern.slf4j.Slf4j;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit4.SpringRunner;

import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

/**
 * @author reghao
 * @date 2023-03-22 16:19:12
 */
@Slf4j
@ActiveProfiles("dev")
@SpringBootTest(classes = DfsStoreApplication.class)
@RunWith(SpringRunner.class)
public class FileMetaTest {
    @Autowired
    FileMetaMapper fileMetaMapper;
    @Autowired
    DataBlockMapper dataBlockMapper;

    /**
     * 正则表达式查询测试
     *
     * @param
     * @return
     * @date 2023-05-23 11:06:07
     */
    public void regexpTest() {
        String bucket = "";
        String prefix = "abc/d/";
        String startAfter = "abc/d/e";
        Integer maxKeys = 10;

        StringBuilder regex = new StringBuilder();
        regex.append("^").append(prefix).append("([^/])+/?$");
        if (startAfter.isBlank()) {
            fileMetaMapper.findAll0(bucket, maxKeys, regex.toString());
        } else {
            fileMetaMapper.findAll2(bucket, prefix, startAfter, maxKeys);
        }
    }

    @Test
    public void test2() {
        int pageSize = 1000;
        int pageNumber = 1;
        Page page = new Page(pageNumber, pageSize);
        List<DataBlock> dataBlocks = dataBlockMapper.findDataBlockByPage(page);
        List<FileMeta> fileMetas = new ArrayList<>();
        while (!dataBlocks.isEmpty()) {
            dataBlocks.forEach(dataBlock -> {
                String objectId = dataBlock.getObjectId();
                String contentId = UUID.randomUUID().toString().replace("-", "");

                FileMeta fileMeta = fileMetaMapper.findByObjectId(objectId);
                if (fileMeta == null) {
                    log.error("{} 不存在", objectId);
                    return;
                }

                fileMeta.setContentId(contentId);
                fileMetas.add(fileMeta);
                dataBlock.setContentId(contentId);
            });

            long start = System.currentTimeMillis();
            if (!fileMetas.isEmpty()) {
                fileMetaMapper.updateBatch(fileMetas);
                fileMetas.clear();
                log.info("batch update FileMeta cost {}", System.currentTimeMillis()-start);
            }

            if (!dataBlocks.isEmpty()) {
                dataBlockMapper.updateBatch(dataBlocks);
                log.info("batch update DataBlock cost {}", System.currentTimeMillis()-start);
            }

            pageNumber++;
            page = new Page(pageNumber, pageSize);
            dataBlocks = dataBlockMapper.findDataBlockByPage(page);
            log.info("page -> {}", pageNumber);
        }
    }

    public void walkDir(Path path) throws IOException {
        Files.walkFileTree(path, new FileVisitor<>() {
            @Override
            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
                return FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                String absolutePath = file.toString();
                try {
                    FileInputStream fis = new FileInputStream(absolutePath);
                    String sha256sum = DigestUtil.sha256sum(fis);
                    FileMeta fileMeta = fileMetaMapper.findBySha256sum(sha256sum);
                    if (fileMeta == null) {
                        log.error("{} 不存在", absolutePath);
                    }
                } catch (Exception ignore) {
                }
                return FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
                return FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
                return FileVisitResult.CONTINUE;
            }
        });
    }

    @Test
    public void test22() throws IOException {
        String baseDir = "/home/reghao/mnt/zdata/porn/0.已完成/";
        Path path = Path.of(baseDir);
        walkDir(path);
    }

    @Test
    public void test11() {
        int pageSize = 10_000;
        int pageNumber = 1;
        Page page = new Page(pageNumber, pageSize);
        List<FileMeta> list = fileMetaMapper.findFileMetaByPage(page);
        while (!list.isEmpty()) {
            list.forEach(fileMeta -> {
                String objectName = fileMeta.getObjectName();
                ObjectMeta objectMeta = fileMetaMapper.findObjectMeta(objectName);
                if (objectMeta == null) {
                    log.error("{} 对应的文件不存在", objectName);
                }
            });

            pageNumber++;
            page = new Page(pageNumber, pageSize);
            list = fileMetaMapper.findFileMetaByPage(page);
            log.info("page -> {}", pageNumber);
        }
    }
}