添加 FileContent, 作为 FileMeta 和 DataBlock 的中间表

dfs-store/src/main/java/cn/reghao/dfs/store/controller/ObjectBasicController.java

@@ -121,11 +121,8 @@ public class ObjectBasicController {
         String contentType = postObject.getContentType();
 
         MultipartFile file = postObject.getFile();
-
         long len = file.getSize();
         InputStream inputStream = file.getInputStream();
-        //String contentType = file.getContentType();
-
         objectBasicService.postObject(objectName, len, contentType, inputStream);
         PostResponse postResponse = new PostResponse();
         return WebResult.success(postResponse);
@@ -144,7 +141,9 @@ public class ObjectBasicController {
 
     @ApiOperation("获取对象")
     @GetMapping(value = "/**")
-    public void getObject() throws IOException {
+    public void getObject(@RequestParam(value = "OSSAccessId", required = false) String ossAccessId,
+                          @RequestParam(value = "Expires", required = false) String expires,
+                          @RequestParam(value = "Signature", required = false) String signature) throws IOException {
         HttpServletRequest servletRequest = ServletUtil.getRequest();
         String uri = servletRequest.getRequestURI();
         String uri1 = URLDecoder.decode(uri, StandardCharsets.UTF_8);

dfs-store/src/main/java/cn/reghao/dfs/store/db/mapper/DataBlockMapper.java

@@ -13,6 +13,7 @@ import java.util.List;
  */
 @Mapper
 public interface DataBlockMapper extends BaseMapper<DataBlock> {
-    void update(@Param("objectId") String objectId, @Param("absolutePath") String absolutePath);
+    @Deprecated
+    void updateContentId(@Param("objectId") String objectId, @Param("contentId") String contentId);
     List<DataBlock> findByObjectId(String objectId);
 }

dfs-store/src/main/java/cn/reghao/dfs/store/db/mapper/FileContentMapper.java

@@ -0,0 +1,14 @@
+package cn.reghao.dfs.store.db.mapper;
+
+import cn.reghao.dfs.store.model.po.FileContent;
+import cn.reghao.jutil.jdk.db.BaseMapper;
+import org.apache.ibatis.annotations.Mapper;
+
+/**
+ * @author reghao
+ * @date 2023-05-10 08:03:35
+ */
+@Mapper
+public interface FileContentMapper extends BaseMapper<FileContent> {
+    String findContentId(String objectId);
+}

dfs-store/src/main/java/cn/reghao/dfs/store/db/mapper/FileMetaMapper.java

@@ -16,8 +16,6 @@ import java.util.List;
  */
 @Mapper
 public interface FileMetaMapper extends BaseMapper<FileMeta> {
-    void update(@Param("objectId") String objectId, @Param("objectName") String objectName);
-
     List<FileMeta> findBySha256sum(String sha256sum);
     FileMeta findByObjectName(String objectName);
     HeadObjectResult findByObjectName1(String objectName);
@@ -25,7 +23,8 @@ public interface FileMetaMapper extends BaseMapper<FileMeta> {
     List<FileMeta> findAll1(@Param("bucket") String bucket, @Param("prefix") String prefix, @Param("max") Integer max);
     List<FileMeta> findAll2(@Param("bucket") String bucket, @Param("prefix") String prefix,
                             @Param("start") String start, @Param("max") Integer max);
-    String findByObjectId(String objectId);
+    @Deprecated
     ObjectMeta findObjectMeta(String objectName);
+    ObjectMeta findObjectMeta1(String objectName);
     List<ObjectMeta> findObjectMetaByPage(Page page);
 }

dfs-store/src/main/java/cn/reghao/dfs/store/model/po/DataBlock.java

@@ -4,6 +4,7 @@ import cn.reghao.jutil.jdk.db.BaseObject;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
+import lombok.Setter;
 
 /**
  * @author reghao
@@ -11,8 +12,10 @@ import lombok.NoArgsConstructor;
  */
 @NoArgsConstructor
 @AllArgsConstructor
+@Setter
 @Getter
 public class DataBlock extends BaseObject<Integer> {
+    private String contentId;
     private String objectId;
     private int index;
     private String blockId;

dfs-store/src/main/java/cn/reghao/dfs/store/model/po/FileContent.java

@@ -0,0 +1,20 @@
+package cn.reghao.dfs.store.model.po;
+
+import cn.reghao.jutil.jdk.db.BaseObject;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+/**
+ * @author reghao
+ * @date 2023-05-10 08:02:44
+ */
+@AllArgsConstructor
+@NoArgsConstructor
+@Setter
+@Getter
+public class FileContent extends BaseObject<Integer> {
+    private String contentId;
+    private String objectId;
+}

dfs-store/src/main/java/cn/reghao/dfs/store/model/po/FileMeta.java

@@ -5,6 +5,8 @@ import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 
+import java.util.UUID;
+
 /**
  * 文件元数据
  *
@@ -26,22 +28,25 @@ public class FileMeta extends BaseObject<Integer> {
     private Integer bucketId;
 
     // 目录对象
-    public FileMeta(String objectName, String objectId, String filename, Integer bucketId) {
+    public FileMeta(String objectName, Integer bucketId) {
         this.objectName = objectName;
-        this.objectId = objectId;
-        this.filename = filename;
+        this.objectId = UUID.randomUUID().toString().replace("-", "");
+        this.filename = "dir";
         this.size = 0L;
         this.fileTypeId = 1000;
+        this.contentType = "0";
+        this.sha256sum = "0";
         this.bucketId = bucketId;
     }
 
-    public FileMeta(String objectName, String filename, FileMeta fileMeta) {
+    public FileMeta(String objectName, String objectId, String filename, FileMeta fileMeta) {
         this.objectName = objectName;
-        this.objectId = fileMeta.getObjectId();
+        this.objectId = objectId;
         this.filename = filename;
-        this.size = 0L;
+        this.size = fileMeta.getSize();
         this.fileTypeId = fileMeta.getFileTypeId();
         this.contentType = fileMeta.getContentType();
+        this.sha256sum = fileMeta.getSha256sum();
         this.bucketId = fileMeta.getBucketId();
     }
 }

dfs-store/src/main/java/cn/reghao/dfs/store/service/ObjectBasicService.java

@@ -2,9 +2,11 @@ package cn.reghao.dfs.store.service;
 
 import cn.reghao.dfs.store.cache.LocalCache;
 import cn.reghao.dfs.store.db.mapper.DataBlockMapper;
+import cn.reghao.dfs.store.db.mapper.FileContentMapper;
 import cn.reghao.dfs.store.db.mapper.FileMetaMapper;
 import cn.reghao.dfs.store.model.po.ContentRange;
 import cn.reghao.dfs.store.model.po.DataBlock;
+import cn.reghao.dfs.store.model.po.FileContent;
 import cn.reghao.dfs.store.model.po.FileMeta;
 import cn.reghao.dfs.store.model.vo.ObjectMeta;
 import cn.reghao.dfs.store.redis.ds.RedisStringObj;
@@ -14,6 +16,7 @@ import cn.reghao.jutil.web.ServletUtil;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.tika.Tika;
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
 import org.springframework.util.StringUtils;
 
 import javax.servlet.http.HttpServletResponse;
@@ -38,6 +41,7 @@ public class ObjectBasicService {
     private final IdGenerator objectIdGenerator;
     private final IdGenerator blockIdGenerator;
     private final FileMetaMapper fileMetaMapper;
+    private FileContentMapper fileContentMapper;
     private final DataBlockMapper dataBlockMapper;
     // 10MiB
     private final int partLength = 1024*1024*10;
@@ -48,13 +52,15 @@ public class ObjectBasicService {
     private final RedisStringObj redisStringObj;
     private Tika tika = new Tika();
 
-    public ObjectBasicService(FileMetaMapper fileMetaMapper, DataBlockMapper dataBlockMapper,
+    public ObjectBasicService(FileMetaMapper fileMetaMapper, FileContentMapper fileContentMapper,
+                              DataBlockMapper dataBlockMapper,
                               FileStoreService fileStoreService, FileUrlService fileUrlService,
                               FileTypeService fileTypeService, LocalCache localCache,
                               RedisStringObj redisStringObj) {
         this.objectIdGenerator = new IdGenerator(32, "object-id");
         this.blockIdGenerator = new IdGenerator(32, "block-id");
         this.fileMetaMapper = fileMetaMapper;
+        this.fileContentMapper = fileContentMapper;
         this.dataBlockMapper = dataBlockMapper;
         this.fileStoreService = fileStoreService;
         this.fileUrlService = fileUrlService;
@@ -63,6 +69,7 @@ public class ObjectBasicService {
         this.redisStringObj = redisStringObj;
     }
 
+    @Transactional(rollbackFor = Exception.class)
     public void putObject(String objectName, File file, String sha256sum) throws Exception {
         String[] names = objectName.split("/");
         String filename = names[names.length-1];
@@ -76,47 +83,62 @@ public class ObjectBasicService {
                 return;
             }
 
-            log.info("暂未实现 PUT 存储");
+            log.info("更新对象操作暂未实现");
             return;
         }
 
+        addParent(objectName);
         List<FileMeta> list = fileMetaMapper.findBySha256sum(sha256sum);
         if (list.isEmpty()) {
             String objectId = objectIdGenerator.stringId();
+            String contentId = UUID.randomUUID().toString().replace("-", "");
+            FileContent fileContent = new FileContent(contentId, objectId);
             long len = file.length();
             List<DataBlock> blocks = store(objectId, len, file);
-
             int fileTypeId = fileTypeService.getFileType1(contentType);
             FileMeta fileMeta = new FileMeta(objectName, objectId, filename, len, fileTypeId, contentType, sha256sum, "tnb", 2);
-            dataBlockMapper.saveAll(blocks);
+
             fileMetaMapper.save(fileMeta);
+            fileContentMapper.save(fileContent);
+            dataBlockMapper.saveAll(blocks);
         } else {
             FileMeta fileMeta = list.get(0);
-            List<DataBlock> blocks = dataBlockMapper.findByObjectId(fileMeta.getObjectId());
+            String contentId = fileContentMapper.findContentId(fileMeta.getObjectId());
+            String objectId = UUID.randomUUID().toString().replace("-", "");
+            FileContent fileContent = new FileContent(contentId, objectId);
+            FileMeta fileMeta1 = new FileMeta(objectName, objectId, filename, fileMeta);
 
-            FileMeta fileMeta1 = new FileMeta(objectName, filename, fileMeta);
             fileMetaMapper.save(fileMeta1);
+            fileContentMapper.save(fileContent);
         }
     }
 
-    public void postObject(String objectName, long len, String contentType, InputStream inputStream) throws Exception {
-        String objectId = objectIdGenerator.stringId();
-        List<DataBlock> list = store(objectId, len, inputStream);
-        String sha256sum = DigestUtil.sha256sum(new FileInputStream(list.get(0).getAbsolutePath()));
-        log.info("{} sha256sum {}", objectName, sha256sum);
-        String[] names = objectName.split("/");
-        String filename = names[names.length-1];
+    private void addParent(String objectName) {
+        List<String> list = getParent(objectName);
+        List<FileMeta> fileMetas = new ArrayList<>();
+        list.forEach(parentName -> {
+            FileMeta fileMeta = fileMetaMapper.findByObjectName(parentName);
+            if (fileMeta == null) {
+                fileMetas.add(new FileMeta(parentName, 2));
+            }
+        });
 
-        /*FileMeta fileMeta = fileMetaMapper.findBySha256sum(sha256sum);
-        if (fileMeta == null) {
-            int fileTypeId = fileTypeService.getFileType1(contentType);
-            fileMeta = new FileMeta(objectName, objectId, filename, len, fileTypeId, contentType, sha256sum, "tnb", 2);
-            dataBlockMapper.saveAll(list);
-            fileMetaMapper.save(fileMeta);
-        } else {
-            FileMeta fileMeta1 = new FileMeta(objectName, filename, fileMeta);
-            fileMetaMapper.save(fileMeta1);
-        }*/
+        if (!fileMetas.isEmpty()) {
+            fileMetaMapper.saveAll(fileMetas);
+        }
+    }
+
+    private List<String> getParent(String objectName) {
+        String[] arr = objectName.split("/");
+        List<String> list = new ArrayList<>();
+        list.add(arr[0] + "/");
+        for (int i = 1; i < arr.length-1; i++) {
+            list.add(list.get(i-1) + arr[i] + "/");
+        }
+        return list;
+    }
+
+    public void postObject(String objectName, long len, String contentType, InputStream inputStream) throws Exception {
     }
 
     private List<DataBlock> store(String objectId, long len, File file) throws IOException {

dfs-store/src/main/resources/mapper/DataBlockMapper.xml

@@ -4,22 +4,22 @@
 <mapper namespace="cn.reghao.dfs.store.db.mapper.DataBlockMapper">
     <insert id="save" useGeneratedKeys="true" keyProperty="id">
         insert into data_block
-        (`id`,`deleted`,`create_time`,`update_time`,`object_id`,`index`,`block_id`,`absolute_path`,`start`,`end`)
+        (`id`,`deleted`,`create_time`,`update_time`,`content_id`,`object_id`,`index`,`block_id`,`absolute_path`,`start`,`end`)
         values
-        (#{id},#{deleted},#{createTime},#{updateTime},#{objectId},#{index},#{blockId},#{absolutePath},#{start},#{end})
+        (#{id},#{deleted},#{createTime},#{updateTime},#{contentId},#{objectId},#{index},#{blockId},#{absolutePath},#{start},#{end})
     </insert>
     <insert id="saveAll" useGeneratedKeys="true" keyProperty="id">
         insert into data_block
-        (`id`,`deleted`,`create_time`,`update_time`,`object_id`,`index`,`block_id`,`absolute_path`,`start`,`end`)
+        (`id`,`deleted`,`create_time`,`update_time`,`content_id`,`object_id`,`index`,`block_id`,`absolute_path`,`start`,`end`)
         values
         <foreach collection="list" item="item" index="index" separator=",">
-            (#{item.id},#{item.deleted},#{item.createTime},#{item.updateTime},#{item.objectId},#{item.index},#{item.blockId},#{item.absolutePath},#{item.start},#{item.end})
+            (#{item.id},#{item.deleted},#{item.createTime},#{item.updateTime},#{item.contentId},#{item.objectId},#{item.index},#{item.blockId},#{item.absolutePath},#{item.start},#{item.end})
         </foreach>
     </insert>
 
-    <update id="update">
+    <update id="updateContentId">
         update data_block
-        set absolute_path=#{absolutePath}
+        set content_id=#{contentId}
         where object_id=#{objectId}
     </update>
 

dfs-store/src/main/resources/mapper/FileContentMapper.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+
+<mapper namespace="cn.reghao.dfs.store.db.mapper.FileContentMapper">
+    <insert id="save" useGeneratedKeys="true" keyProperty="id">
+        insert into file_content
+        (`id`,`content_id`,`object_id`)
+        values
+        (#{id},#{deleted},#{createTime},#{updateTime},#{contentId},#{objectId})
+    </insert>
+    <insert id="saveAll" useGeneratedKeys="true" keyProperty="id">
+        insert into file_content
+        (`id`,`content_id`,`object_id`)
+        values
+        <foreach collection="list" item="item" index="index" separator=",">
+            (#{item.id},#{item.deleted},#{item.createTime},#{item.updateTime},#{item.contentId},#{item.objectId})
+        </foreach>
+    </insert>
+
+    <select id="findContentId" resultType="java.lang.String">
+        select content_id from file_content where object_id=#{objectId}
+    </select>
+</mapper>

dfs-store/src/main/resources/mapper/FileMetaMapper.xml

@@ -13,7 +13,7 @@
         (`id`,`deleted`,`create_time`,`update_time`,`object_name`,`object_id`,`filename`,`size`,`file_type_id`,`content_type`,`sha256sum`,`bucket_id`)
         values
         <foreach collection="list" item="item" index="index" separator=",">
-            (#{item.id},#{item.deleted},#{item.createTime},#{item.updateTime},#{item.objectName},#{item.objectId},#{item.filename},#{item.size},#{item.fileTypeId},#{item.contentType},#{item.sha256sum},#{item.bucket_id})
+            (#{item.id},#{item.deleted},#{item.createTime},#{item.updateTime},#{item.objectName},#{item.objectId},#{item.filename},#{item.size},#{item.fileTypeId},#{item.contentType},#{item.sha256sum},#{item.bucketId})
         </foreach>
     </insert>
 
@@ -77,6 +77,15 @@
         on file_meta.object_id=data_block.object_id
         where file_meta.object_name=#{objectName}
     </select>
+    <select id="findObjectMeta1" resultType="cn.reghao.dfs.store.model.vo.ObjectMeta">
+        select file_meta.size,file_meta.content_type,file_meta.object_name,file_meta.object_id,
+        data_block.absolute_path
+        from file_meta
+        inner join file_content
+        inner join data_block
+        on file_meta.object_id=file_content.object_id and file_content.content_id=data_block.content_id
+        where file_meta.object_name=#{objectName}
+    </select>
     <select id="findObjectMetaByPage" resultType="cn.reghao.dfs.store.model.vo.ObjectMeta">
         select file_meta.size,file_meta.content_type,file_meta.object_name,data_block.absolute_path
         from file_meta

dfs-store/src/test/java/FileTest.java

@@ -1,5 +1,9 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.tika.Tika;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
 
 /**
  * @author reghao
@@ -7,6 +11,25 @@ import org.apache.tika.Tika;
  */
 @Slf4j
 public class FileTest {
+    @Test
+    public void test4() {
+        String objectName = "home/reghao/Downloads/public.sql";
+        objectName = "home/reghao";
+        List<String> list = getParent(objectName);
+        System.out.println();
+        //FileMeta fileMeta = fileMetaMapper.findByObjectName(objectName);
+    }
+
+    List<String> getParent(String objectName) {
+        String[] arr = objectName.split("/");
+        List<String> list = new ArrayList<>();
+        list.add(arr[0] + "/");
+        for (int i = 1; i < arr.length-1; i++) {
+            list.add(list.get(i-1) + arr[i] + "/");
+        }
+        return list;
+    }
+
     public static void main(String[] args) {
         String filePath = "/home/reghao/Downloads/public.sql";
 

dfs-store/src/test/java/RedisTest.java

@@ -1,7 +1,9 @@
 import cn.reghao.dfs.store.DfsStoreApplication;
 import cn.reghao.dfs.store.db.mapper.DataBlockMapper;
+import cn.reghao.dfs.store.db.mapper.FileContentMapper;
 import cn.reghao.dfs.store.db.mapper.FileMetaMapper;
 import cn.reghao.dfs.store.model.po.DataBlock;
+import cn.reghao.dfs.store.model.po.FileContent;
 import cn.reghao.dfs.store.model.po.FileMeta;
 import cn.reghao.dfs.store.model.vo.ObjectMeta;
 import cn.reghao.dfs.store.redis.ds.RedisString;
@@ -15,7 +17,9 @@ import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.ActiveProfiles;
 import org.springframework.test.context.junit4.SpringRunner;
 
+import java.util.ArrayList;
 import java.util.List;
+import java.util.UUID;
 
 /**
  * @author reghao
@@ -71,4 +75,89 @@ public class RedisTest {
             log.info("page -> {}", pageNumber);
         }
     }
+
+    @Autowired
+    FileContentMapper fileContentMapper;
+    @Test
+    public void test2() {
+        List<FileMeta> list = fileMetaMapper.findAll();
+        list.forEach(fileMeta -> {
+            String objectId = fileMeta.getObjectId();
+            List<DataBlock> list1 = dataBlockMapper.findByObjectId(objectId);
+            if (list1.isEmpty()) {
+                log.info("{} 为空", objectId);
+                return;
+            }
+
+            DataBlock dataBlock = list1.get(0);
+            String contentId = UUID.randomUUID().toString().replace("-", "");
+            FileContent fileContent = new FileContent(contentId, objectId);
+            dataBlock.setContentId(contentId);
+
+            fileContentMapper.save(fileContent);
+            dataBlockMapper.updateContentId(objectId, contentId);
+        });
+    }
+
+    @Test
+    public void test3() {
+        List<FileMeta> list = fileMetaMapper.findAll();
+        list.forEach(fileMeta -> {
+            String objectId = fileMeta.getObjectId();
+            String objectName = fileMeta.getObjectName();
+            ObjectMeta objectMeta = fileMetaMapper.findObjectMeta1(objectName);
+            System.out.println();
+        });
+    }
+
+    @Test
+    public void test4() {
+        String objectName = "audio/playback/rZ7EKPbG41.mp3";
+        List<String> list = getParent(objectName);
+        List<FileMeta> fileMetas = new ArrayList<>();
+        list.forEach(parentName -> {
+            FileMeta fileMeta = fileMetaMapper.findByObjectName(parentName);
+            if (fileMeta == null) {
+                fileMetas.add(new FileMeta(parentName, 2));
+            }
+        });
+
+        if (!fileMetas.isEmpty()) {
+            fileMetaMapper.saveAll(fileMetas);
+        }
+    }
+
+    void addParent(String objectName) {
+        List<String> list = getParent(objectName);
+        List<FileMeta> fileMetas = new ArrayList<>();
+        list.forEach(parentName -> {
+            FileMeta fileMeta = fileMetaMapper.findByObjectName(parentName);
+            if (fileMeta == null) {
+                fileMetas.add(new FileMeta(parentName, 2));
+            }
+        });
+
+        if (!fileMetas.isEmpty()) {
+            fileMetaMapper.saveAll(fileMetas);
+        }
+    }
+
+    List<String> getParent(String objectName) {
+        String[] arr = objectName.split("/");
+        List<String> list = new ArrayList<>();
+        list.add(arr[0] + "/");
+        for (int i = 1; i < arr.length-1; i++) {
+            list.add(list.get(i-1) + arr[i] + "/");
+        }
+        return list;
+    }
+
+    private String getUrl(String objectName) {
+        String domain = "https://oss.reghao.cn/";
+        StringBuilder sb = new StringBuilder();
+        sb.append("?").append("Expires=").append("")
+                .append("&OSSAccessId=").append("")
+                .append("&Signature=").append("");
+        return domain + objectName + sb.toString();
+    }
 }