update

dfs-store/src/main/java/cn/reghao/dfs/store/db/mapper/DataBlockMapper.java

@@ -2,7 +2,6 @@ package cn.reghao.dfs.store.db.mapper;
 
 import cn.reghao.dfs.store.model.po.DataBlock;
 import cn.reghao.jutil.jdk.db.BaseMapper;
-import cn.reghao.jutil.jdk.db.Page;
 import org.apache.ibatis.annotations.Mapper;
 import org.apache.ibatis.annotations.Param;
 
@@ -19,9 +18,6 @@ public interface DataBlockMapper extends BaseMapper<DataBlock> {
     @Deprecated
     void updateBatch(List<DataBlock> list);
 
-    @Deprecated
-    List<DataBlock> findByObjectId(String objectId);
     List<DataBlock> findByContentId(String contentId);
-    @Deprecated
-    DataBlock findByPath(String absolutePath);
+    List<DataBlock> findDataBlocks(@Param("pageSize") int pageSize, @Param("nextId") int nextId);
 }

dfs-store/src/main/resources/application-dev.yml

@@ -1,16 +1,16 @@
 dubbo:
   registry:
-    address: zookeeper://127.0.0.1:2181
+    address: zookeeper://192.168.0.110:2181
 spring:
   redis:
     database: 0
-    host: 127.0.0.1
+    host: 192.168.0.110
     port: 6379
-    password: Dev@123456
+    password: Test@123456
   datasource:
-    url: jdbc:mysql://127.0.0.1:3306/reghao_oss_rdb?useSSL=false&useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2b8
-    username: dev
-    password: Dev@123456
+    url: jdbc:mysql://192.168.0.110:3306/reghao_oss_tdb?useSSL=false&useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2b8
+    username: test
+    password: Test@123456
 oss:
   domain: oss.reghao.cn
   mountedDirs:

dfs-store/src/main/resources/mapper/DataBlockMapper.xml

@@ -42,17 +42,15 @@
     <select id="findAll" resultType="cn.reghao.dfs.store.model.po.DataBlock">
         select * from data_block
     </select>
-    <select id="findByObjectId" resultType="cn.reghao.dfs.store.model.po.DataBlock">
-        select * from data_block
-        where object_id=#{objectId}
-    </select>
     <select id="findByContentId" resultType="cn.reghao.dfs.store.model.po.DataBlock">
         select *
         from data_block
         where content_id=#{contentId}
     </select>
-    <select id="findByPath" resultType="cn.reghao.dfs.store.model.po.DataBlock">
-        select * from data_block
-        where absolute_path=#{absolutePath}
+    <select id="findDataBlocks" resultType="cn.reghao.dfs.store.model.po.DataBlock">
+        select *
+        from data_block
+        where id>#{nextId}
+        limit #{pageSize}
     </select>
 </mapper>

dfs-store/src/test/java/FileMetaTest.java

@@ -1,5 +1,7 @@
 import cn.reghao.dfs.store.DfsStoreApplication;
+import cn.reghao.dfs.store.db.mapper.DataBlockMapper;
 import cn.reghao.dfs.store.db.mapper.FileMetaMapper;
+import cn.reghao.dfs.store.model.po.DataBlock;
 import cn.reghao.dfs.store.model.po.FileMeta;
 import cn.reghao.dfs.store.service.ObjectNameService;
 import cn.reghao.jutil.jdk.db.Page;
@@ -7,6 +9,7 @@ import cn.reghao.jutil.jdk.security.DigestUtil;
 import cn.reghao.jutil.jdk.thread.ThreadPoolWrapper;
 import cn.reghao.oss.api.constant.ObjectACL;
 import cn.reghao.oss.api.constant.UploadChannel;
+import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.FileUtils;
 import org.junit.Test;
@@ -165,4 +168,30 @@ public class FileMetaTest {
             objectNameService.createParentDirs(objectName, ObjectACL.PUBLIC.getCode());
         }
     }
+
+    @Autowired
+    DataBlockMapper dataBlockMapper;
+    @Test
+    public void dataBlockTest() {
+        int pageSize = 10000;
+        int nextId = 0;
+        List<DataBlock> list = dataBlockMapper.findDataBlocks(pageSize, nextId);
+        while (!list.isEmpty()) {
+            process(list);
+
+            nextId = list.get(list.size()-1).getId();
+            list = dataBlockMapper.findDataBlocks(pageSize, nextId);
+            log.info("nextId -> {}", nextId);
+        }
+    }
+
+    private void process(List<DataBlock> list) {
+        for (DataBlock dataBlock : list) {
+            String contentId = dataBlock.getContentId();
+            List<FileMeta> list1 = fileMetaMapper.findByContentId(contentId);
+            if (list1.isEmpty()) {
+                log.info("{} not exist in file_meta", contentId);
+            }
+        }
+    }
 }

dfs-store/src/test/java/FileTest.java

@@ -7,8 +7,10 @@ import cn.reghao.jutil.media.model.MediaProps;
 import cn.reghao.jutil.media.model.VideoProps;
 import cn.reghao.jutil.jdk.security.DigestUtil;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
 import org.junit.Test;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.file.FileVisitResult;
 import java.nio.file.FileVisitor;
@@ -36,8 +38,7 @@ public class FileTest {
             @Override
             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                 String absolutePath = file.toString();
-                //process(absolutePath);
-                unique(absolutePath);
+                process(absolutePath);
                 return FileVisitResult.CONTINUE;
             }
 
@@ -54,6 +55,13 @@ public class FileTest {
     }
 
     private void process(String absolutePath) {
+        String mediaType = FileType.getMediaType(absolutePath);
+        if (mediaType.startsWith("text")) {
+            FileUtils.deleteQuietly(new File(absolutePath));
+        }
+    }
+
+    private void getMediaInfo(String absolutePath) {
         MediaProps mediaProps = FFmpegWrapper.getMediaProps(absolutePath);
         if (mediaProps == null) {
             log.error("{} 没有媒体信息", absolutePath);
@@ -102,15 +110,14 @@ public class FileTest {
 
     @Test
     public void test22() throws IOException {
-        //String baseDir = "/home/reghao/mnt/porn/1.待发布/13996.反差婊系列/p/";
-        //Path path = Path.of(baseDir);
-        //walkDir(path);
+        String baseDir = "/home/reghao/Downloads/b/";
+        Path path = Path.of(baseDir);
+        walkDir(path);
 
-        String filePath = "/home/reghao/Downloads/仁和区 2.m4a";
+        /*String filePath = "/home/reghao/Downloads/仁和区 2.m4a";
         filePath = "/home/reghao/Downloads/三生三世.wav";
         String mediaType = FileType.getMediaType(filePath);
         MediaProps mediaProps = FFmpegWrapper.getMediaProps(filePath);
-        //FFmpegWrapper.convertAudio(filePath, "/home/reghao/Downloads/三生三世.m4a");
-        System.out.println();
+        FFmpegWrapper.convertAudio(filePath, "/home/reghao/Downloads/三生三世.m4a");*/
     }
 }

dfs-store/src/test/java/ImageFileTest.java

@@ -1,29 +0,0 @@
-import cn.reghao.dfs.store.DfsStoreApplication;
-import cn.reghao.dfs.store.db.mapper.ImageFileMapper;
-import cn.reghao.dfs.store.model.po.ImageFile;
-import lombok.extern.slf4j.Slf4j;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.test.context.ActiveProfiles;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import java.util.List;
-
-/**
- * @author reghao
- * @date 2023-05-31 22:25:21
- */
-@Slf4j
-@ActiveProfiles("dev")
-@SpringBootTest(classes = DfsStoreApplication.class)
-@RunWith(SpringRunner.class)
-public class ImageFileTest {
-    @Autowired
-    ImageFileMapper imageFileMapper;
-
-    @Test
-    public void imageTest() {
-    }
-}

dfs-store/src/test/java/RedisTest.java

@@ -1,9 +1,6 @@
 import cn.reghao.dfs.store.DfsStoreApplication;
-import cn.reghao.dfs.store.db.mapper.DataBlockMapper;
 import cn.reghao.dfs.store.db.mapper.FileMetaMapper;
-import cn.reghao.dfs.store.model.po.DataBlock;
 import cn.reghao.dfs.store.model.po.FileMeta;
-import cn.reghao.jutil.jdk.security.DigestUtil;
 import cn.reghao.oss.api.dto.ObjectMeta;
 import cn.reghao.dfs.store.redis.ds.RedisStringObj;
 import cn.reghao.jutil.jdk.db.Page;
@@ -15,9 +12,6 @@ import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.ActiveProfiles;
 import org.springframework.test.context.junit4.SpringRunner;
 
-import java.io.File;
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
 import java.util.List;
 
 /**
@@ -54,13 +48,6 @@ public class RedisTest {
         List<ObjectMeta> list = fileMetaMapper.findObjectMetaByPage(page);
         while (!list.isEmpty()) {
             list.forEach(objectMeta -> {
-                /*List<DataBlock> list1 = dataBlockMapper.findByObjectId(objectMeta.getObjectId());
-                if (list1.isEmpty()) {
-                    return;
-                }*
-                String absolutePath = list1.get(0).getAbsolutePath();
-                objectMeta.setAbsolutePath(absolutePath);*/
-
                 String objectName = objectMeta.getObjectName();
                 String key1 = key + objectName;
                 redisString.set(key1, objectMeta);
@@ -72,21 +59,4 @@ public class RedisTest {
             log.info("page -> {}", pageNumber);
         }
     }
-
-    @Autowired
-    DataBlockMapper dataBlockMapper;
-    private void checkLocalFile(File file) throws IOException, NoSuchAlgorithmException {
-        log.error("--------------------------------------");
-        String absolutePath = file.getAbsolutePath();
-        DataBlock dataBlock = dataBlockMapper.findByPath(absolutePath);
-        if (dataBlock == null) {
-            log.error("{} not exist in data_block", absolutePath);
-        }
-
-        String sha256sum = DigestUtil.sha256sum(absolutePath);
-        FileMeta fileMeta = fileMetaMapper.findBySha256sum(sha256sum);
-        if (fileMeta == null) {
-            log.error("{} not exist in file_meta", absolutePath);
-        }
-    }
 }