|
|
@@ -2,13 +2,7 @@ import cn.reghao.oss.store.OssStoreApplication;
|
|
|
import cn.reghao.oss.store.db.mapper.DataBlockMapper;
|
|
|
import cn.reghao.oss.store.db.mapper.FileMetaMapper;
|
|
|
import cn.reghao.oss.store.model.po.DataBlock;
|
|
|
-import cn.reghao.oss.store.model.po.FileMeta;
|
|
|
-import cn.reghao.jutil.jdk.store.SubDirCount;
|
|
|
-import cn.reghao.jutil.jdk.db.Page;
|
|
|
-import cn.reghao.jutil.jdk.security.DigestUtil;
|
|
|
-import cn.reghao.jutil.jdk.thread.ThreadPoolWrapper;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
-import org.apache.commons.io.FileUtils;
|
|
|
import org.junit.Test;
|
|
|
import org.junit.runner.RunWith;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
@@ -16,16 +10,7 @@ import org.springframework.boot.test.context.SpringBootTest;
|
|
|
import org.springframework.test.context.ActiveProfiles;
|
|
|
import org.springframework.test.context.junit4.SpringRunner;
|
|
|
|
|
|
-import java.io.File;
|
|
|
-import java.io.IOException;
|
|
|
-import java.nio.file.FileVisitResult;
|
|
|
-import java.nio.file.FileVisitor;
|
|
|
-import java.nio.file.Files;
|
|
|
-import java.nio.file.Path;
|
|
|
-import java.nio.file.attribute.BasicFileAttributes;
|
|
|
-import java.security.NoSuchAlgorithmException;
|
|
|
import java.util.List;
|
|
|
-import java.util.concurrent.ExecutorService;
|
|
|
|
|
|
/**
|
|
|
* @author reghao
|
|
|
@@ -55,102 +40,6 @@ public class FileMetaTest {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- /**
|
|
|
- * 分页处理
|
|
|
- *
|
|
|
- * @param
|
|
|
- * @return
|
|
|
- * @date 2023-05-30 00:26:07
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void processByPage() {
|
|
|
- int pageSize = 10_000;
|
|
|
- int pageNumber = 1;
|
|
|
- Page page = new Page(pageNumber, pageSize);
|
|
|
- List<FileMeta> list = fileMetaMapper.findFileMetaByPage(page);
|
|
|
- while (!list.isEmpty()) {
|
|
|
- list.forEach(fileMeta -> {
|
|
|
-
|
|
|
- });
|
|
|
-
|
|
|
- pageNumber++;
|
|
|
- page = new Page(pageNumber, pageSize);
|
|
|
- list = fileMetaMapper.findFileMetaByPage(page);
|
|
|
- log.info("page -> {}", pageNumber);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- public void walkDir(Path path) throws IOException {
|
|
|
- Files.walkFileTree(path, new FileVisitor<>() {
|
|
|
- @Override
|
|
|
- public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
|
|
|
- return FileVisitResult.CONTINUE;
|
|
|
- }
|
|
|
-
|
|
|
- @Override
|
|
|
- public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
|
|
|
- String absolutePath = file.toString();
|
|
|
- process(absolutePath);
|
|
|
- return FileVisitResult.CONTINUE;
|
|
|
- }
|
|
|
-
|
|
|
- @Override
|
|
|
- public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
|
|
|
- return FileVisitResult.CONTINUE;
|
|
|
- }
|
|
|
-
|
|
|
- @Override
|
|
|
- public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
|
|
|
- return FileVisitResult.CONTINUE;
|
|
|
- }
|
|
|
- });
|
|
|
- }
|
|
|
-
|
|
|
- private void process(String absolutePath) {
|
|
|
- try {
|
|
|
- String sha256sum = DigestUtil.sha256sum(absolutePath);
|
|
|
- FileMeta fileMeta = fileMetaMapper.findBySha256sum(sha256sum);
|
|
|
- if (fileMeta != null) {
|
|
|
- FileUtils.deleteQuietly(new File(absolutePath));
|
|
|
- log.info("{} 删除 {} -> {}", Thread.currentThread().getName(), total++, absolutePath);
|
|
|
- } else {
|
|
|
- //log.info("{} 在 db 中不存在", absolutePath);
|
|
|
- }
|
|
|
- } catch (IOException | NoSuchAlgorithmException e) {
|
|
|
- e.printStackTrace();
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- ExecutorService threadPool = ThreadPoolWrapper.threadPool("delete-pool", 10);
|
|
|
- int total = 1;
|
|
|
- @Test
|
|
|
- public void deleteFile() throws IOException, InterruptedException {
|
|
|
- String baseDir = "";
|
|
|
- Path path = Path.of(baseDir);
|
|
|
- //walkDir(path);
|
|
|
-
|
|
|
- threadPool.submit(new Task(baseDir));
|
|
|
- Thread.sleep(3600_000*24*7);
|
|
|
- }
|
|
|
-
|
|
|
- class Task implements Runnable {
|
|
|
- private final String baseDir;
|
|
|
-
|
|
|
- public Task(String baseDir) {
|
|
|
- this.baseDir = baseDir;
|
|
|
- }
|
|
|
-
|
|
|
- @Override
|
|
|
- public void run() {
|
|
|
- Path path = Path.of(baseDir);
|
|
|
- try {
|
|
|
- walkDir(path);
|
|
|
- } catch (IOException e) {
|
|
|
- e.printStackTrace();
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
@Autowired
|
|
|
DataBlockMapper dataBlockMapper;
|
|
|
@Test
|
|
|
@@ -159,46 +48,11 @@ public class FileMetaTest {
|
|
|
int nextId = 0;
|
|
|
List<DataBlock> list = dataBlockMapper.findDataBlocks(pageSize, nextId);
|
|
|
while (!list.isEmpty()) {
|
|
|
- process1(list);
|
|
|
- dataBlockMapper.updateBatch(list);
|
|
|
+ // process DataBlocks
|
|
|
|
|
|
nextId = list.get(list.size()-1).getId();
|
|
|
list = dataBlockMapper.findDataBlocks(pageSize, nextId);
|
|
|
log.info("nextId -> {}", nextId);
|
|
|
}
|
|
|
}
|
|
|
-
|
|
|
- private void process(List<DataBlock> list) {
|
|
|
- for (DataBlock dataBlock : list) {
|
|
|
- String contentId = dataBlock.getContentId();
|
|
|
- List<FileMeta> list1 = fileMetaMapper.findByContentId(contentId);
|
|
|
- if (list1.isEmpty()) {
|
|
|
- log.info("{} not exist in file_meta", contentId);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- private void process1(List<DataBlock> list) {
|
|
|
- for (DataBlock dataBlock : list) {
|
|
|
- int id = dataBlock.getId();
|
|
|
- String absolutePath = dataBlock.getAbsolutePath();
|
|
|
- int idx = absolutePath.lastIndexOf("/");
|
|
|
- String parent = absolutePath.substring(0, idx);
|
|
|
- String relativeDir = parent.replace("", "");
|
|
|
- //dataBlock.setRelativeDir(relativeDir);
|
|
|
-
|
|
|
- /*String contentId = dataBlock.getContentId();
|
|
|
- List<FileMeta> list1 = fileMetaMapper.findByContentId(contentId);
|
|
|
- if (list1.isEmpty()) {
|
|
|
- log.info("{} not exist in file_meta", contentId);
|
|
|
- }*/
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- @Test
|
|
|
- public void test123() {
|
|
|
- String baseDir = "";
|
|
|
- List<SubDirCount> list = dataBlockMapper.findSubDirCount("");
|
|
|
- System.out.println();
|
|
|
- }
|
|
|
}
|