oss/dfs-store/src/test/java/FileMetaTest.java

import cn.reghao.dfs.store.DfsStoreApplication;
import cn.reghao.dfs.store.db.mapper.*;
import cn.reghao.dfs.store.model.po.*;
import cn.reghao.dfs.store.service.ObjectNameService;
import cn.reghao.dfs.store.util.UserContext;
import cn.reghao.jutil.jdk.db.Page;
import cn.reghao.jutil.jdk.security.DigestUtil;
import cn.reghao.jutil.jdk.thread.ThreadPoolWrapper;
import cn.reghao.oss.api.constant.ObjectScope;
import cn.reghao.oss.api.constant.UploadChannel;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit4.SpringRunner;

import java.io.File;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.security.NoSuchAlgorithmException;
import java.util.List;
import java.util.concurrent.ExecutorService;

/**
 * @author reghao
 * @date 2023-07-16 19:19:12
 */
@Slf4j
@ActiveProfiles("dev")
@SpringBootTest(classes = DfsStoreApplication.class)
@RunWith(SpringRunner.class)
public class FileMetaTest {
    @Autowired
    FileMetaMapper fileMetaMapper;

    // 正则表达式查询测试
    public void regexpTest() {
        String bucket = "";
        String prefix = "abc/d/";
        String startAfter = "abc/d/e";
        Integer maxKeys = 10;

        StringBuilder regex = new StringBuilder();
        regex.append("^").append(prefix).append("([^/])+/?$");
        if (startAfter.isBlank()) {
            fileMetaMapper.findAll0(bucket, maxKeys, regex.toString());
        } else {
            fileMetaMapper.findAll2(bucket, prefix, startAfter, maxKeys);
        }
    }

    /**
     * 分页处理
     *
     * @param
     * @return
     * @date 2023-05-30 00:26:07
     */
    @Test
    public void processByPage() {
        int pageSize = 10_000;
        int pageNumber = 1;
        Page page = new Page(pageNumber, pageSize);
        List<FileMeta> list = fileMetaMapper.findFileMetaByPage(page);
        while (!list.isEmpty()) {
            list.forEach(fileMeta -> {

            });

            pageNumber++;
            page = new Page(pageNumber, pageSize);
            list = fileMetaMapper.findFileMetaByPage(page);
            log.info("page -> {}", pageNumber);
        }
    }

    public void walkDir(Path path) throws IOException {
        Files.walkFileTree(path, new FileVisitor<>() {
            @Override
            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
                return FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                String absolutePath = file.toString();
                process(absolutePath);
                return FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
                return FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
                return FileVisitResult.CONTINUE;
            }
        });
    }

    private void process(String absolutePath) {
        try {
            String sha256sum = DigestUtil.sha256sum(absolutePath);
            FileMeta fileMeta = fileMetaMapper.findBySha256sum(sha256sum);
            if (fileMeta != null) {
                FileUtils.deleteQuietly(new File(absolutePath));
                log.info("{} 删除 {} -> {}", Thread.currentThread().getName(), total++, absolutePath);
            } else {
                //log.info("{} 在 db 中不存在", absolutePath);
            }
        } catch (IOException | NoSuchAlgorithmException e) {
            e.printStackTrace();
        }
    }

    ExecutorService threadPool = ThreadPoolWrapper.threadPool("delete-pool", 10);
    int total = 1;
    @Test
    public void deleteFile() throws IOException, InterruptedException {
        String baseDir = "/home/reghao/mnt1/spider/bili/img/vidcover";
        //baseDir = "/run/media/reghao/7c43343d-c1e2-4e68-bc18-89bd3f61b0c8/cover";
        //baseDir = "/home/reghao/mnt1/spider/bili/img/avatar";
        Path path = Path.of(baseDir);
        //walkDir(path);

        threadPool.submit(new Task("/home/reghao/mnt1/spider/bili/img/avatar"));
        threadPool.submit(new Task("/run/media/reghao/7c43343d-c1e2-4e68-bc18-89bd3f61b0c8/cover"));
        threadPool.submit(new Task("/run/media/reghao/7c43343d-c1e2-4e68-bc18-89bd3f61b0c8/avatar"));

        Thread.sleep(3600_000*24*7);
    }

    class Task implements Runnable {
        private final String baseDir;

        public Task(String baseDir) {
            this.baseDir = baseDir;
        }

        @Override
        public void run() {
            Path path = Path.of(baseDir);
            try {
                walkDir(path);
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

    @Autowired
    ObjectNameService objectNameService;
    @Test
    public void initUploadChannel() {
        UserContext userContext = new UserContext(10001L);
        for (UploadChannel channel : UploadChannel.values()) {
            String objectName = channel.getPrefix();
            objectNameService.createParentDirs(objectName, ObjectScope.PUBLIC.getCode());
        }
    }

    @Autowired
    DataBlockMapper dataBlockMapper;
    @Test
    public void dataBlockTest() {
        int pageSize = 10000;
        int nextId = 0;
        List<DataBlock> list = dataBlockMapper.findDataBlocks(pageSize, nextId);
        while (!list.isEmpty()) {
            process(list);

            nextId = list.get(list.size()-1).getId();
            list = dataBlockMapper.findDataBlocks(pageSize, nextId);
            log.info("nextId -> {}", nextId);
        }
    }

    private void process(List<DataBlock> list) {
        for (DataBlock dataBlock : list) {
            String contentId = dataBlock.getContentId();
            List<FileMeta> list1 = fileMetaMapper.findByContentId(contentId);
            if (list1.isEmpty()) {
                log.info("{} not exist in file_meta", contentId);
            }
        }
    }
}