更新分片上传接口

dfs-store/src/main/java/cn/reghao/dfs/store/controller/MultipartUploadController.java

@@ -0,0 +1,65 @@
+package cn.reghao.dfs.store.controller;
+
+import cn.reghao.dfs.store.model.dto.*;
+import cn.reghao.dfs.store.service.MultipartUploadService;
+import cn.reghao.jutil.jdk.result.WebResult;
+import io.swagger.annotations.ApiOperation;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.http.MediaType;
+import org.springframework.validation.annotation.Validated;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author reghao
+ * @date 2023-05-11 17:45:02
+ */
+@RestController
+@Slf4j
+public class MultipartUploadController {
+    private final MultipartUploadService multipartUploadService;
+
+    public MultipartUploadController(MultipartUploadService multipartUploadService) {
+        this.multipartUploadService = multipartUploadService;
+    }
+
+    @ApiOperation("创建分片上传")
+    @PostMapping(value = "/", params = {"create"}, produces = MediaType.APPLICATION_JSON_VALUE)
+    public String videoFilePrepare(@Validated UploadPrepare uploadPrepare) {
+        log.info("创建文件分片上传");
+        UploadPrepareRet uploadPrepareRet = multipartUploadService.prepareUpload(uploadPrepare);
+        return WebResult.success(uploadPrepareRet);
+    }
+
+    @ApiOperation(value = "获取已上传的文件分片")
+    @GetMapping(value = "/", params = {"multipart"}, produces = MediaType.APPLICATION_JSON_VALUE)
+    public String uploadedVideoFilePart() {
+        log.info("返回已上传的文件分片");
+        Map<String, Object> map = new HashMap<>();
+        map.put("skipUpload", false);
+        map.put("url", "");
+        map.put("needMerge", false);
+        map.put("uploaded", Collections.emptyList());
+        return WebResult.success(map);
+    }
+
+    @ApiOperation(value = "上传分片文件")
+    @PostMapping(value = "/", params = {"multipart"}, produces = MediaType.APPLICATION_JSON_VALUE)
+    public String uploadVideoFile0(@Validated UploadFilePart uploadFilePart) throws Exception {
+        log.info("上传的文件分片");
+        FilePartRet uploadedFileRet = multipartUploadService.putFilePart(uploadFilePart);
+        return WebResult.success(uploadedFileRet);
+    }
+
+    @ApiOperation(value = "合并分片文件")
+    @PostMapping(value = "/", params = {"merge"}, produces = MediaType.APPLICATION_JSON_VALUE)
+    public String merge() {
+        log.info("合并上传的文件分片");
+        return WebResult.success();
+    }
+}

dfs-store/src/main/java/cn/reghao/dfs/store/db/mapper/DataBlockMapper.java

@@ -2,6 +2,7 @@ package cn.reghao.dfs.store.db.mapper;
 
 import cn.reghao.dfs.store.model.po.DataBlock;
 import cn.reghao.jutil.jdk.db.BaseMapper;
+import cn.reghao.jutil.jdk.db.Page;
 import org.apache.ibatis.annotations.Mapper;
 import org.apache.ibatis.annotations.Param;
 
@@ -16,4 +17,5 @@ public interface DataBlockMapper extends BaseMapper<DataBlock> {
     @Deprecated
     void updateContentId(@Param("objectId") String objectId, @Param("contentId") String contentId);
     List<DataBlock> findByObjectId(String objectId);
+    List<DataBlock> findDataBlockByPage(Page page);
 }

dfs-store/src/main/java/cn/reghao/dfs/store/db/mapper/FileMetaMapper.java

@@ -16,7 +16,7 @@ import java.util.List;
  */
 @Mapper
 public interface FileMetaMapper extends BaseMapper<FileMeta> {
-    List<FileMeta> findBySha256sum(String sha256sum);
+    FileMeta findBySha256sum(String sha256sum);
     FileMeta findByObjectName(String objectName);
     HeadObjectResult findByObjectName1(String objectName);
     List<FileMeta> findAll0(@Param("bucket") String bucket, @Param("max") Integer max, @Param("regex") String regex);

dfs-store/src/main/java/cn/reghao/dfs/store/model/dto/FilePartRet.java

@@ -0,0 +1,25 @@
+package cn.reghao.dfs.store.model.dto;
+
+import lombok.Getter;
+
+/**
+ * @author reghao
+ * @date 2022-04-21 09:32:00
+ */
+@Getter
+public class FilePartRet {
+    private final String uploadId;
+    private final boolean merged;
+    private String url;
+
+    public FilePartRet(String uploadId) {
+        this.uploadId = uploadId;
+        this.merged = false;
+    }
+
+    public FilePartRet(String uploadId, String url) {
+        this.uploadId = uploadId;
+        this.merged = true;
+        this.url = url;
+    }
+}

dfs-store/src/main/java/cn/reghao/dfs/store/model/dto/PathUrl.java

@@ -2,7 +2,6 @@ package cn.reghao.dfs.store.model.dto;
 
 import lombok.AllArgsConstructor;
 import lombok.Getter;
-import lombok.NoArgsConstructor;
 
 /**
  * 文件的本地路径和 URL
@@ -11,12 +10,9 @@ import lombok.NoArgsConstructor;
  * @date 2022-04-26 15:10:04
  */
 @AllArgsConstructor
-@NoArgsConstructor
 @Getter
 public class PathUrl {
+    private String contentId;
     // 本地文件路径
     private String absolutePath;
-    private String url;
-    // 不包括域名的路径
-    private String path;
 }

dfs-store/src/main/java/cn/reghao/dfs/store/model/dto/UploadFilePart.java

@@ -0,0 +1,44 @@
+package cn.reghao.dfs.store.model.dto;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.web.multipart.MultipartFile;
+
+import javax.validation.constraints.NotBlank;
+import javax.validation.constraints.NotNull;
+import java.io.Serializable;
+
+/**
+ * @author reghao
+ * @date 2022-04-25 10:42:38
+ */
+@Getter
+@Setter
+public class UploadFilePart implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+    private String key;
+    @NotNull
+    private MultipartFile file;
+    // 文件标识
+    @NotBlank
+    private String identifier;
+    private String filename;
+    private String relativePath;
+    // 文件大小
+    @NotNull
+    private Long totalSize;
+    // 分片文件大小
+    @NotNull
+    private Long chunkSize;
+    // 当前分片文件大小
+    @NotNull
+    private Integer currentChunkSize;
+    // 分片文件数量
+    @NotNull
+    private Integer totalChunks;
+    // 当前分片文件索引
+    @NotNull
+    private Integer chunkNumber;
+    private String partSha256sum;
+}

dfs-store/src/main/java/cn/reghao/dfs/store/model/dto/UploadPrepare.java

@@ -0,0 +1,27 @@
+package cn.reghao.dfs.store.model.dto;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import javax.validation.constraints.NotBlank;
+import javax.validation.constraints.NotNull;
+import java.io.Serializable;
+
+/**
+ * 分片文件
+ *
+ * @author reghao
+ * @date 2021-11-23 10:23:00
+ */
+@Setter
+@Getter
+public class UploadPrepare implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+    @NotBlank
+    private String filename;
+    @NotNull
+    private Long size;
+    @NotBlank
+    private String sha256sum;
+}

dfs-store/src/main/java/cn/reghao/dfs/store/model/dto/UploadPrepareRet.java

@@ -0,0 +1,17 @@
+package cn.reghao.dfs.store.model.dto;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+/**
+ * @author reghao
+ * @date 2022-04-21 09:27:55
+ */
+@AllArgsConstructor
+@Getter
+public class UploadPrepareRet {
+    private String uploadId;
+    private long splitSize;
+    private boolean exist;
+    private String url;
+}

dfs-store/src/main/java/cn/reghao/dfs/store/model/dto/UploadedFileRet.java

@@ -0,0 +1,15 @@
+package cn.reghao.dfs.store.model.dto;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+/**
+ * @author reghao
+ * @date 2022-07-23 06:21:32
+ */
+@AllArgsConstructor
+@Getter
+public class UploadedFileRet {
+    private String videoFileId;
+    private boolean merged;
+}

dfs-store/src/main/java/cn/reghao/dfs/store/model/po/DataBlock.java

@@ -16,10 +16,10 @@ import lombok.Setter;
 @Getter
 public class DataBlock extends BaseObject<Integer> {
     private String contentId;
+    @Deprecated
     private String objectId;
     private int index;
     private String blockId;
-    //private long size;
     private String absolutePath;
     private long start;
     private long end;

dfs-store/src/main/java/cn/reghao/dfs/store/model/po/FileMeta.java

@@ -14,7 +14,6 @@ import java.util.UUID;
  * @date 2022-11-21 10:53:10
  */
 @NoArgsConstructor
-@AllArgsConstructor
 @Getter
 public class FileMeta extends BaseObject<Integer> {
     private String objectName;
@@ -24,19 +23,17 @@ public class FileMeta extends BaseObject<Integer> {
     private Integer fileTypeId;
     private String contentType;
     private String sha256sum;
-    private String bucket;
     private Integer bucketId;
 
-    // 目录对象
-    public FileMeta(String objectName, Integer bucketId) {
+    public FileMeta(String objectName, String objectId, String filename, long size, String contentType, String sha256sum) {
         this.objectName = objectName;
-        this.objectId = UUID.randomUUID().toString().replace("-", "");
-        this.filename = "dir";
-        this.size = 0L;
-        this.fileTypeId = 1000;
-        this.contentType = "0";
-        this.sha256sum = "0";
-        this.bucketId = bucketId;
+        this.objectId = objectId;
+        this.filename = filename;
+        this.size = size;
+        this.fileTypeId = 1001;
+        this.contentType = contentType;
+        this.sha256sum = sha256sum;
+        this.bucketId = 2;
     }
 
     public FileMeta(String objectName, String objectId, String filename, FileMeta fileMeta) {
@@ -49,4 +46,16 @@ public class FileMeta extends BaseObject<Integer> {
         this.sha256sum = fileMeta.getSha256sum();
         this.bucketId = fileMeta.getBucketId();
     }
+
+    // 目录对象
+    public FileMeta(String objectName) {
+        this.objectName = objectName;
+        this.objectId = UUID.randomUUID().toString().replace("-", "");
+        this.filename = "dir";
+        this.size = 0L;
+        this.fileTypeId = 1000;
+        this.contentType = "dir";
+        this.sha256sum = "0";
+        this.bucketId = 2;
+    }
 }

dfs-store/src/main/java/cn/reghao/dfs/store/rpc/ObjectServiceImpl.java

@@ -117,7 +117,7 @@ public class ObjectServiceImpl implements ObjectService {
             String objectId = objectIdGenerator.stringId();
             String[] names1 = objectName1.split("/");
             String filename = names1[names1.length-1];
-            list.add(new FileMeta(objectName1, 1));
+            list.add(new FileMeta(objectName1));
         }
 
         if (!list.isEmpty()) {

dfs-store/src/main/java/cn/reghao/dfs/store/service/FileTypeService.java

@@ -1,81 +0,0 @@
-package cn.reghao.dfs.store.service;
-
-import cn.reghao.dfs.store.model.dto.FileContentType;
-import cn.reghao.dfs.store.util.StringUtil;
-import org.springframework.stereotype.Service;
-
-import java.io.File;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-
-/**
- * @author reghao
- * @date 2022-04-26 16:33:09
- */
-@Service
-public class FileTypeService {
-    public FileContentType getFileType(String filename) {
-        String suffix = StringUtil.getSuffix(filename);
-
-        int fileType = 4;
-        String contentType = "application/octet-stream";
-        if (suffix.isBlank()) {
-            return new FileContentType(suffix, fileType, contentType);
-        }
-
-        if (suffix.startsWith("jpg")) {
-            fileType = 1;
-            contentType = "image/jpg";
-        } else if (suffix.startsWith("gif")) {
-            fileType = 1;
-            contentType = "image/gif";
-        } else if (suffix.startsWith("png")) {
-            fileType = 1;
-            contentType = "image/png";
-        } else if (suffix.startsWith("mp3")) {
-            fileType = 2;
-            contentType = "audio/mp3";
-        } else if (suffix.startsWith("mp4")) {
-            fileType = 3;
-            contentType = "video/mp4";
-        }
-        return new FileContentType(suffix, fileType, contentType);
-    }
-
-    public FileContentType getFileType(String contentType, String filePath) throws Exception {
-        if (contentType == null) {
-            contentType = Files.probeContentType(Paths.get(filePath));
-        }
-
-        int fileType = 4;
-        if (contentType == null) {
-            contentType = "application/octet-stream";
-        } else if (contentType.startsWith("image")) {
-            fileType = 1;
-        } else if (contentType.startsWith("audio")) {
-            fileType = 2;
-        } else if (contentType.startsWith("video")) {
-            fileType = 3;
-        }
-
-        String filename = filePath.substring(filePath.lastIndexOf(File.separator)+1);
-        String suffix = StringUtil.getSuffix(filename);
-        return new FileContentType(suffix, fileType, contentType);
-    }
-
-    public Integer getFileType1(String contentType) {
-        int fileType = 1005;
-        if (contentType == null) {
-            return fileType;
-        } else if (contentType.startsWith("image")) {
-            fileType = 1001;
-        } else if (contentType.startsWith("video")) {
-            fileType = 1002;
-        } else if (contentType.startsWith("audio")) {
-            fileType = 1003;
-        } else if (contentType.startsWith("text")) {
-            fileType = 1004;
-        }
-        return fileType;
-    }
-}

dfs-store/src/main/java/cn/reghao/dfs/store/service/FileUrlService.java

@@ -1,14 +1,10 @@
 package cn.reghao.dfs.store.service;
 
 import cn.reghao.dfs.store.config.DfsProperties;
-import cn.reghao.dfs.store.model.dto.PathUrl;
 import cn.reghao.dfs.store.util.store.LoadBalancer;
 import cn.reghao.dfs.store.util.store.StoreDir;
 import org.springframework.stereotype.Service;
 
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
-
 /**
  * @author reghao
  * @date 2022-04-26 15:06:29
@@ -27,41 +23,10 @@ public class FileUrlService {
         this.loadBalancer = loadBalancer;
     }
 
-    public PathUrl genPathUrl(String contentType, String sha256sum, long fileSize, String fileId,
-                              String suffix, String uploadId) throws IOException, NoSuchAlgorithmException {
-        String path;
-        if (contentType == null) {
-            path = String.format("file/%s.%s", uploadId, suffix);
-        } else if (contentType.startsWith("image")) {
-            path = String.format("image/%s.%s", uploadId, suffix);
-        } else if (contentType.startsWith("video")) {
-            path = String.format("video/playback/%s.%s", uploadId, suffix);
-        } else {
-            path = String.format("file/%s.%s", uploadId, suffix);
-        }
-
-        return genPathAndUrl(sha256sum, fileSize, fileId, suffix, path);
-    }
-
-    private PathUrl genPathAndUrl(String sha256sum, long fileSize, String fileId, String suffix, String path)
-            throws IOException, NoSuchAlgorithmException {
-        //StoreDir storeDir = loadBalancer.getStoreDir(fileSize, sha256sum);
-        StoreDir storeDir = loadBalancer.getStoreDir(fileSize);
-        String fileDir = storeDir.getAbsoluteDirPath();
-
-        String filePath = String.format("%s/%s.%s", fileDir, fileId, suffix);
-        String url = String.format("//%s/%s", domain, path);
-        return new PathUrl(filePath, url, path);
-    }
-
-    public String genFilePath(long len, String objectId, String suffix) {
-        StoreDir storeDir = loadBalancer.getStoreDir(len);
+    public String genFilePath(long size, String objectId, String suffix) {
+        //StoreDir storeDir = loadBalancer.getStoreDir(size, sha256sum);
+        StoreDir storeDir = loadBalancer.getStoreDir(size);
         String fileDir = storeDir.getAbsoluteDirPath();
         return String.format("%s/%s.%s", fileDir, objectId, suffix);
     }
-
-    public StoreDir getStoreDir(String sha256sum, long fileSize) {
-        //StoreDir storeDir = loadBalancer.getStoreDir(fileSize, sha256sum);
-        return loadBalancer.getStoreDir(fileSize);
-    }
 }

dfs-store/src/main/java/cn/reghao/dfs/store/service/MultipartUploadService.java

@@ -0,0 +1,131 @@
+package cn.reghao.dfs.store.service;
+
+import cn.reghao.dfs.store.db.mapper.DataBlockMapper;
+import cn.reghao.dfs.store.db.mapper.FileContentMapper;
+import cn.reghao.dfs.store.db.mapper.FileMetaMapper;
+import cn.reghao.dfs.store.model.dto.*;
+import cn.reghao.dfs.store.model.po.DataBlock;
+import cn.reghao.dfs.store.model.po.FileContent;
+import cn.reghao.dfs.store.model.po.FileMeta;
+import cn.reghao.jutil.jdk.security.DigestUtil;
+import cn.reghao.jutil.jdk.shell.Shell;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * @author reghao
+ * @date 2021-08-04 17:29:47
+ */
+@Slf4j
+@Service
+public class MultipartUploadService {
+    // 20MiB
+    private final static long PART_SIZE = 1024*1024*20;
+    private final FileMetaMapper fileMetaMapper;
+    private final FileContentMapper fileContentMapper;
+    private final DataBlockMapper dataBlockMapper;
+    private final FileUrlService fileUrlService;
+    private final FileStoreService fileStoreService;
+    private final Map<String, Set<Integer>> map = new HashMap<>();
+    private final Map<String, PathUrl> pathMap = new HashMap<>();
+
+    public MultipartUploadService(FileMetaMapper fileMetaMapper, FileContentMapper fileContentMapper,
+                                  DataBlockMapper dataBlockMapper,
+                                  FileUrlService fileUrlService, FileStoreService fileStoreService) {
+        this.fileMetaMapper = fileMetaMapper;
+        this.fileContentMapper = fileContentMapper;
+        this.dataBlockMapper = dataBlockMapper;
+        this.fileUrlService = fileUrlService;
+        this.fileStoreService = fileStoreService;
+    }
+
+    public synchronized UploadPrepareRet prepareUpload(UploadPrepare uploadPrepare) {
+        String sha256sum = uploadPrepare.getSha256sum();
+        FileMeta fileMeta = fileMetaMapper.findBySha256sum(sha256sum);
+        if (fileMeta == null) {
+            return new UploadPrepareRet("uploadId", PART_SIZE, false, null);
+        } else {
+            String objectName = fileMeta.getObjectName();
+            String url = String.format("https://file.reghao.cn/%s", objectName);
+            return new UploadPrepareRet("uploadId", PART_SIZE, false, url);
+        }
+    }
+
+    /**
+     * 处理通过 HTTP 请求提交的分片文件
+     *
+     * @param
+     * @return
+     * @date 2021-12-08 下午3:31
+     */
+    @Transactional(rollbackFor = Exception.class)
+    public synchronized FilePartRet putFilePart(UploadFilePart uploadFilePart) throws Exception {
+        String partSha256sum = uploadFilePart.getPartSha256sum();
+        long totalSize = uploadFilePart.getTotalSize();
+        long chunkSize = uploadFilePart.getChunkSize();
+        int currentPartSize = uploadFilePart.getCurrentChunkSize();
+        int totalParts = uploadFilePart.getTotalChunks();
+        int chunkNumber = uploadFilePart.getChunkNumber();
+        MultipartFile multipartFile = uploadFilePart.getFile();
+        log.info("{} -> {}:{}", currentPartSize, totalParts, chunkNumber);
+
+        String uploadId = uploadFilePart.getIdentifier();
+        Set<Integer> set = map.computeIfAbsent(uploadId, k -> new HashSet<>());
+        if (set.isEmpty()) {
+            String contentId = UUID.randomUUID().toString().replace("-", "");
+            String absolutePath = fileUrlService.genFilePath(totalSize, contentId, "dat");
+            PathUrl pathUrl = new PathUrl(contentId, absolutePath);
+            pathMap.put(uploadId, pathUrl);
+            fileStoreService.createSparseFile(absolutePath, totalSize);
+        }
+
+        if (set.add(chunkNumber)) {
+            long pos = (chunkNumber-1) * chunkSize;
+            String absolutePath = pathMap.get(uploadId).getAbsolutePath();
+            fileStoreService.writeToFile(multipartFile.getInputStream(), absolutePath, pos);
+        }
+
+        if (set.size() != totalParts) {
+            return new FilePartRet(uploadId);
+        } else {
+            String absolutePath = pathMap.get(uploadId).getAbsolutePath();
+            FileInputStream fis = new FileInputStream(absolutePath);
+            String sha256sumMerged = DigestUtil.sha256sum(fis);
+            log.info("合并的文件 {}", absolutePath);
+            if (!uploadId.equals(sha256sumMerged)) {
+                throw new Exception("分片合并文件的 sha256sum 与原文件不一致!");
+            }
+
+            String contentId = pathMap.get(uploadId).getContentId();
+            String objectName = uploadFilePart.getKey();
+            String objectId = UUID.randomUUID().toString().replace("-", "");
+            String filename = uploadFilePart.getFilename();
+            String contentType = getMediaType(absolutePath);
+            FileMeta fileMeta = new FileMeta(objectName, objectId, filename, totalSize, contentType, sha256sumMerged);
+            FileContent fileContent = new FileContent(contentId, objectId);
+            List<DataBlock> blocks = new ArrayList<>();
+            String blockId = UUID.randomUUID().toString();
+            blocks.add(new DataBlock(contentId, 0, blockId, absolutePath));
+
+            fileMetaMapper.save(fileMeta);
+            fileContentMapper.save(fileContent);
+            dataBlockMapper.saveAll(blocks);
+
+            map.remove(uploadId);
+            pathMap.remove(uploadId);
+            String url = String.format("https://file.reghao.cn/%s", objectName);
+            log.info("url -> {}", url);
+            return new FilePartRet(uploadId, url);
+        }
+    }
+
+    private String getMediaType(String src) {
+        String cmd = String.format("/bin/file -b --mime-type \"%s\"", src);
+        return Shell.execWithResult(cmd);
+    }
+}

dfs-store/src/main/java/cn/reghao/dfs/store/service/PutObjectService.java

@@ -9,15 +9,11 @@ import cn.reghao.dfs.store.model.po.FileContent;
 import cn.reghao.dfs.store.model.po.FileMeta;
 import cn.reghao.dfs.store.redis.ds.RedisStringObj;
 import cn.reghao.jutil.jdk.shell.Shell;
-import cn.reghao.jutil.tool.id.IdGenerator;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
 import java.io.*;
-import java.nio.channels.FileChannel;
-import java.nio.channels.WritableByteChannel;
-import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.UUID;
@@ -29,30 +25,23 @@ import java.util.UUID;
 @Slf4j
 @Service
 public class PutObjectService {
-    private final IdGenerator objectIdGenerator;
-    private final IdGenerator blockIdGenerator;
     private final FileMetaMapper fileMetaMapper;
-    private FileContentMapper fileContentMapper;
+    private final FileContentMapper fileContentMapper;
     private final DataBlockMapper dataBlockMapper;
-    private final FileStoreService fileStoreService;
     private final FileUrlService fileUrlService;
-    private final FileTypeService fileTypeService;
+    private final FileStoreService fileStoreService;
     private final LocalCache localCache;
     private final RedisStringObj redisStringObj;
 
     public PutObjectService(FileMetaMapper fileMetaMapper, FileContentMapper fileContentMapper,
                             DataBlockMapper dataBlockMapper,
                             FileStoreService fileStoreService, FileUrlService fileUrlService,
-                            FileTypeService fileTypeService, LocalCache localCache,
-                            RedisStringObj redisStringObj) {
-        this.objectIdGenerator = new IdGenerator(32, "object-id");
-        this.blockIdGenerator = new IdGenerator(32, "block-id");
+                            LocalCache localCache, RedisStringObj redisStringObj) {
         this.fileMetaMapper = fileMetaMapper;
         this.fileContentMapper = fileContentMapper;
         this.dataBlockMapper = dataBlockMapper;
         this.fileStoreService = fileStoreService;
         this.fileUrlService = fileUrlService;
-        this.fileTypeService = fileTypeService;
         this.localCache = localCache;
         this.redisStringObj = redisStringObj;
     }
@@ -76,21 +65,21 @@ public class PutObjectService {
         }
 
         addParent(objectName);
-        List<FileMeta> list = fileMetaMapper.findBySha256sum(sha256sum);
-        if (list.isEmpty()) {
-            String objectId = objectIdGenerator.stringId();
+        FileMeta fileMeta = fileMetaMapper.findBySha256sum(sha256sum);
+        if (fileMeta == null) {
+            String objectId = UUID.randomUUID().toString().replace("-", "");
             String contentId = UUID.randomUUID().toString().replace("-", "");
             FileContent fileContent = new FileContent(contentId, objectId);
-            long len = file.length();
-            List<DataBlock> blocks = store(contentId, len, file);
-            int fileTypeId = fileTypeService.getFileType1(contentType);
-            FileMeta fileMeta = new FileMeta(objectName, objectId, filename, len, fileTypeId, contentType, sha256sum, "tnb", 2);
+            long size = file.length();
+            FileInputStream fis = new FileInputStream(file);
+            List<DataBlock> blocks = store(contentId, size, fis);
+            file.delete();
+            fileMeta = new FileMeta(objectName, objectId, filename, size, contentType, sha256sum);
 
             fileMetaMapper.save(fileMeta);
             fileContentMapper.save(fileContent);
             dataBlockMapper.saveAll(blocks);
         } else {
-            FileMeta fileMeta = list.get(0);
             String contentId = fileContentMapper.findContentId(fileMeta.getObjectId());
             String objectId = UUID.randomUUID().toString().replace("-", "");
             FileContent fileContent = new FileContent(contentId, objectId);
@@ -112,7 +101,7 @@ public class PutObjectService {
         list.forEach(parentName -> {
             FileMeta fileMeta = fileMetaMapper.findByObjectName(parentName);
             if (fileMeta == null) {
-                fileMetas.add(new FileMeta(parentName, 2));
+                fileMetas.add(new FileMeta(parentName));
             }
         });
 
@@ -131,33 +120,13 @@ public class PutObjectService {
         return list;
     }
 
-    public void postObject(String objectName, long len, String contentType, InputStream inputStream) throws Exception {
-
-    }
-
-    private List<DataBlock> store(String contentId, long len, File file) throws IOException {
-        FileInputStream fis = new FileInputStream(file);
-        FileChannel inputChannel = fis.getChannel();
-
+    private List<DataBlock> store(String contentId, long size, InputStream inputStream) throws IOException {
         List<DataBlock> list = new ArrayList<>();
-        String absolutePath = fileUrlService.genFilePath(len, contentId, "dat");
-        FileOutputStream fos = new FileOutputStream(absolutePath);
-        WritableByteChannel targetChannel = fos.getChannel();
-        inputChannel.transferTo(0, inputChannel.size(), targetChannel);
-        Files.delete(file.toPath());
-
-        String blockId = UUID.randomUUID().toString();
-        list.add(new DataBlock(contentId, 0, blockId, absolutePath));
-        return list;
-    }
-
-    private List<DataBlock> store(String objectId, long len, InputStream inputStream) throws IOException {
-        List<DataBlock> list = new ArrayList<>();
-        String absolutePath = fileUrlService.genFilePath(len, objectId, "dat");
+        String absolutePath = fileUrlService.genFilePath(size, contentId, "dat");
         fileStoreService.saveFile(absolutePath, inputStream);
 
         String blockId = UUID.randomUUID().toString();
-        list.add(new DataBlock(objectId, 0, blockId, absolutePath));
+        list.add(new DataBlock(contentId, 0, blockId, absolutePath));
         return list;
     }
 

dfs-store/src/main/resources/mapper/DataBlockMapper.xml

@@ -4,16 +4,16 @@
 <mapper namespace="cn.reghao.dfs.store.db.mapper.DataBlockMapper">
     <insert id="save" useGeneratedKeys="true" keyProperty="id">
         insert into data_block
-        (`id`,`deleted`,`create_time`,`update_time`,`content_id`,`object_id`,`index`,`block_id`,`absolute_path`,`start`,`end`)
+        (`id`,`deleted`,`create_time`,`update_time`,`content_id`,`index`,`block_id`,`absolute_path`,`start`,`end`)
         values
-        (#{id},#{deleted},#{createTime},#{updateTime},#{contentId},#{objectId},#{index},#{blockId},#{absolutePath},#{start},#{end})
+        (#{id},#{deleted},#{createTime},#{updateTime},#{contentId},#{index},#{blockId},#{absolutePath},#{start},#{end})
     </insert>
     <insert id="saveAll" useGeneratedKeys="true" keyProperty="id">
         insert into data_block
-        (`id`,`deleted`,`create_time`,`update_time`,`content_id`,`object_id`,`index`,`block_id`,`absolute_path`,`start`,`end`)
+        (`id`,`deleted`,`create_time`,`update_time`,`content_id`,`index`,`block_id`,`absolute_path`,`start`,`end`)
         values
         <foreach collection="list" item="item" index="index" separator=",">
-            (#{item.id},#{item.deleted},#{item.createTime},#{item.updateTime},#{item.contentId},#{item.objectId},#{item.index},#{item.blockId},#{item.absolutePath},#{item.start},#{item.end})
+            (#{item.id},#{item.deleted},#{item.createTime},#{item.updateTime},#{item.contentId},#{item.index},#{item.blockId},#{item.absolutePath},#{item.start},#{item.end})
         </foreach>
     </insert>
 
@@ -30,4 +30,10 @@
         select * from data_block
         where object_id=#{objectId}
     </select>
+    <select id="findDataBlockByPage" resultType="cn.reghao.dfs.store.model.po.DataBlock">
+        select data_block.*
+        from data_block
+        inner join file_meta
+        on data_block.object_id=file_meta.object_id
+    </select>
 </mapper>

dfs-store/src/main/resources/mapper/FileMetaMapper.xml

@@ -56,6 +56,8 @@
     <select id="findBySha256sum" resultType="cn.reghao.dfs.store.model.po.FileMeta">
         select * from file_meta
         where sha256sum=#{sha256sum}
+        order by create_time asc
+        limit 1
     </select>
     <select id="findByObjectName" resultType="cn.reghao.dfs.store.model.po.FileMeta">
         select * from file_meta

dfs-store/src/test/java/RedisTest.java

@@ -80,84 +80,28 @@ public class RedisTest {
     FileContentMapper fileContentMapper;
     @Test
     public void test2() {
-        List<FileMeta> list = fileMetaMapper.findAll();
-        list.forEach(fileMeta -> {
-            String objectId = fileMeta.getObjectId();
-            List<DataBlock> list1 = dataBlockMapper.findByObjectId(objectId);
-            if (list1.isEmpty()) {
-                log.info("{} 为空", objectId);
-                return;
-            }
-
-            DataBlock dataBlock = list1.get(0);
-            String contentId = UUID.randomUUID().toString().replace("-", "");
-            FileContent fileContent = new FileContent(contentId, objectId);
-            dataBlock.setContentId(contentId);
-
-            fileContentMapper.save(fileContent);
-            dataBlockMapper.updateContentId(objectId, contentId);
-        });
-    }
-
-    @Test
-    public void test3() {
-        List<FileMeta> list = fileMetaMapper.findAll();
-        list.forEach(fileMeta -> {
-            String objectId = fileMeta.getObjectId();
-            String objectName = fileMeta.getObjectName();
-            ObjectMeta objectMeta = fileMetaMapper.findObjectMeta1(objectName);
-            System.out.println();
-        });
-    }
-
-    @Test
-    public void test4() {
-        String objectName = "audio/playback/rZ7EKPbG41.mp3";
-        List<String> list = getParent(objectName);
-        List<FileMeta> fileMetas = new ArrayList<>();
-        list.forEach(parentName -> {
-            FileMeta fileMeta = fileMetaMapper.findByObjectName(parentName);
-            if (fileMeta == null) {
-                fileMetas.add(new FileMeta(parentName, 2));
-            }
-        });
-
-        if (!fileMetas.isEmpty()) {
-            fileMetaMapper.saveAll(fileMetas);
-        }
-    }
+        int pageSize = 10000;
+        int pageNumber = 1;
+        Page page = new Page(pageNumber, pageSize);
+        List<DataBlock> dataBlocks = dataBlockMapper.findDataBlockByPage(page);
+        while (!dataBlocks.isEmpty()) {
+            List<FileContent> list = new ArrayList<>();
+            dataBlocks.forEach(dataBlock -> {
+                String objectId = dataBlock.getObjectId();
+                String contentId = UUID.randomUUID().toString().replace("-", "");
+                list.add(new FileContent(contentId, objectId));
+                dataBlock.setContentId(contentId);
+                dataBlockMapper.updateContentId(objectId, contentId);
+            });
 
-    void addParent(String objectName) {
-        List<String> list = getParent(objectName);
-        List<FileMeta> fileMetas = new ArrayList<>();
-        list.forEach(parentName -> {
-            FileMeta fileMeta = fileMetaMapper.findByObjectName(parentName);
-            if (fileMeta == null) {
-                fileMetas.add(new FileMeta(parentName, 2));
+            if (!list.isEmpty()) {
+                fileContentMapper.saveAll(list);
             }
-        });
 
-        if (!fileMetas.isEmpty()) {
-            fileMetaMapper.saveAll(fileMetas);
-        }
-    }
-
-    List<String> getParent(String objectName) {
-        String[] arr = objectName.split("/");
-        List<String> list = new ArrayList<>();
-        list.add(arr[0] + "/");
-        for (int i = 1; i < arr.length-1; i++) {
-            list.add(list.get(i-1) + arr[i] + "/");
+            pageNumber++;
+            page = new Page(pageNumber, pageSize);
+            dataBlocks = dataBlockMapper.findDataBlockByPage(page);
+            log.info("page -> {}", pageNumber);
         }
-        return list;
-    }
-
-    private String getUrl(String objectName) {
-        String domain = "https://oss.reghao.cn/";
-        StringBuilder sb = new StringBuilder();
-        sb.append("?").append("Expires=").append("")
-                .append("&OSSAccessId=").append("")
-                .append("&Signature=").append("");
-        return domain + objectName + sb.toString();
     }
 }