优化 PUT 方法上传文件

README.md

@@ -3,3 +3,4 @@
 
 ## 依赖的第三方服务
 - mysql
+- zookeeper

src/main/java/cn/reghao/dfs/store/config/FileMessageConverter.java

@@ -8,10 +8,7 @@ import org.springframework.http.converter.AbstractHttpMessageConverter;
 import org.springframework.http.converter.HttpMessageNotReadableException;
 import org.springframework.http.converter.HttpMessageNotWritableException;
 
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
+import java.io.*;
 
 /**
  * 实现 PUT 请求上传文件
@@ -21,7 +18,7 @@ import java.io.InputStream;
 @Slf4j
 public class FileMessageConverter extends AbstractHttpMessageConverter<File> {
     public FileMessageConverter() {
-        super(MediaType.APPLICATION_OCTET_STREAM);
+        super(MediaType.ALL);
     }
 
     @Override
@@ -34,19 +31,16 @@ public class FileMessageConverter extends AbstractHttpMessageConverter<File> {
             throws IOException, HttpMessageNotReadableException {
 
         InputStream inputStream = inputMessage.getBody();
-        File tmpFile = File.createTempFile("upload","tmp");
+        File tmpFile = File.createTempFile("upload",".dat");
         if (inputStream != null) {
             FileOutputStream outputStream = new FileOutputStream(tmpFile);
 
             byte[] buffer = new byte[1024];
             int bytesRead;
-
             while ((bytesRead = inputStream.read(buffer)) > 0) {
                 outputStream.write(buffer, 0, bytesRead);
             }
-
             outputStream.flush();
-
             outputStream.close();
         }
         return tmpFile;

src/main/java/cn/reghao/dfs/store/controller/ObjectBasicController.java

@@ -19,6 +19,8 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.URLDecoder;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
 
 /**
  * @author reghao
@@ -42,11 +44,10 @@ public class ObjectBasicController {
         String uri1 = URLDecoder.decode(uri, StandardCharsets.UTF_8);
 
         String objectName = uri1.replaceFirst("/", "");
-        long len = 0;
-        String contentType = "";
-        FileInputStream fis = new FileInputStream(file);
+        String contentType = servletRequest.getContentType();
+        String sha256sum = servletRequest.getHeader("Content-Sha256sum");
 
-        objectBasicService.putObject(objectName, len, contentType, fis);
+        objectBasicService.putObject(objectName, file, contentType, sha256sum);
         return WebBody.success();
     }
 
@@ -54,11 +55,11 @@ public class ObjectBasicController {
     @PostMapping(value = "/")
     public String postObject(@Validated PostObject postObject) throws Exception {
         String key = postObject.getKey();
+        String contentType = postObject.getContentType();
         MultipartFile file = postObject.getFile();
 
         String objectName = key;
         long len = file.getSize();
-        String contentType = file.getContentType();
         InputStream inputStream = file.getInputStream();
 
         objectBasicService.putObject(objectName, len, contentType, inputStream);

src/main/java/cn/reghao/dfs/store/model/po/FileMeta.java

@@ -18,18 +18,19 @@ public class FileMeta extends BaseObject<Integer> {
     private String objectName;
     private String objectId;
     private String filename;
-    private long size;
+    private Long size;
     private Integer fileTypeId;
     private String contentType;
     private String sha256sum;
     private String bucket;
+    private Integer bucketId;
 
     // 目录对象
     public FileMeta(String objectName, String objectId, String filename, String bucket) {
         this.objectName = objectName;
         this.objectId = objectId;
         this.filename = filename;
-        this.size = 0;
+        this.size = 0L;
         this.fileTypeId = 1000;
         this.bucket = bucket;
     }

src/main/java/cn/reghao/dfs/store/service/ObjectBasicService.java

@@ -8,12 +8,15 @@ import cn.reghao.dfs.store.model.po.FileMeta;
 import cn.reghao.jutil.jdk.security.DigestUtil;
 import cn.reghao.jutil.tool.id.IdGenerator;
 import cn.reghao.jutil.web.ServletUtil;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
 import org.springframework.util.StringUtils;
 
 import javax.servlet.http.HttpServletResponse;
 import java.io.*;
 import java.net.URLEncoder;
+import java.nio.channels.FileChannel;
+import java.nio.channels.WritableByteChannel;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
@@ -23,6 +26,7 @@ import java.util.UUID;
  * @author reghao
  * @date 2022-11-23 09:40:18
  */
+@Slf4j
 @Service
 public class ObjectBasicService {
     private final IdGenerator objectIdGenerator;
@@ -49,22 +53,60 @@ public class ObjectBasicService {
         this.fileTypeService = fileTypeService;
     }
 
+    public void putObject(String objectName, File file, String contentType, String sha256sum) throws Exception {
+        String objectId = objectIdGenerator.stringId();
+        long len = file.length();
+
+        List<DataBlock> list = store(objectId, len, file);
+        FileMeta fileMeta = fileMetaMapper.findBySha256sum(sha256sum);
+        if (fileMeta == null) {
+            String[] names = objectName.split("/");
+            String filename = names[names.length-1];
+            int fileTypeId = fileTypeService.getFileType1(contentType);
+
+            fileMeta = new FileMeta(objectName, objectId, filename, len, fileTypeId, contentType, sha256sum, "tnb", 2);
+            dataBlockMapper.saveAll(list);
+            fileMetaMapper.save(fileMeta);
+        } else {
+            log.info("{} sha256sum {} exist", objectName, sha256sum);
+        }
+    }
+
     public void putObject(String objectName, long len, String contentType, InputStream inputStream) throws Exception {
         String objectId = objectIdGenerator.stringId();
         List<DataBlock> list = store(objectId, len, inputStream);
         String sha256sum = DigestUtil.sha256sum(new FileInputStream(list.get(0).getAbsolutePath()));
+        log.info("{} sha256sum {}", objectName, sha256sum);
+
         FileMeta fileMeta = fileMetaMapper.findBySha256sum(sha256sum);
         if (fileMeta == null) {
             String[] names = objectName.split("/");
             String filename = names[names.length-1];
             int fileTypeId = fileTypeService.getFileType1(contentType);
 
-            fileMeta = new FileMeta(objectName, objectId, filename, len, fileTypeId, contentType, sha256sum, "tnb");
+            fileMeta = new FileMeta(objectName, objectId, filename, len, fileTypeId, contentType, sha256sum, "tnb", 2);
             dataBlockMapper.saveAll(list);
             fileMetaMapper.save(fileMeta);
+        } else {
+            log.info("sha256sum {} exist", sha256sum);
         }
     }
 
+    private List<DataBlock> store(String objectId, long len, File file) throws IOException {
+        FileInputStream fis = new FileInputStream(file);
+        FileChannel inputChannel = fis.getChannel();
+
+        List<DataBlock> list = new ArrayList<>();
+        String absolutePath = fileUrlService.genFilePath(len, objectId, "dat");
+        FileOutputStream fos = new FileOutputStream(absolutePath);
+        WritableByteChannel targetChannel = fos.getChannel();
+        inputChannel.transferTo(0, inputChannel.size(), targetChannel);
+
+        String blockId = UUID.randomUUID().toString();
+        list.add(new DataBlock(objectId, 0, blockId, absolutePath));
+        return list;
+    }
+
     private List<DataBlock> store(String objectId, long len, InputStream inputStream) throws IOException {
         List<DataBlock> list = new ArrayList<>();
         String absolutePath = fileUrlService.genFilePath(len, objectId, "dat");

src/main/resources/mapper/oss/FileMetaMapper.xml

@@ -4,16 +4,16 @@
 <mapper namespace="cn.reghao.dfs.store.db.mapper.FileMetaMapper">
     <insert id="save" useGeneratedKeys="true" keyProperty="id">
         insert into file_meta
-        (`id`,`deleted`,`create_time`,`update_time`,`object_name`,`object_id`,`filename`,`size`,`file_type_id`,`content_type`,`sha256sum`,`bucket`)
+        (`id`,`deleted`,`create_time`,`update_time`,`object_name`,`object_id`,`filename`,`size`,`file_type_id`,`content_type`,`sha256sum`,`bucket`,`bucket_id`)
         values
-        (#{id},#{deleted},#{createTime},#{updateTime},#{objectName},#{objectId},#{filename},#{size},#{fileTypeId},#{contentType},#{sha256sum},#{bucket})
+        (#{id},#{deleted},#{createTime},#{updateTime},#{objectName},#{objectId},#{filename},#{size},#{fileTypeId},#{contentType},#{sha256sum},#{bucket},#{bucketId})
     </insert>
     <insert id="saveAll" useGeneratedKeys="true" keyProperty="id">
         insert into file_meta
-        (`id`,`deleted`,`create_time`,`update_time`,`object_name`,`object_id`,`filename`,`size`,`file_type_id`,`content_type`,`sha256sum`,`bucket`)
+        (`id`,`deleted`,`create_time`,`update_time`,`object_name`,`object_id`,`filename`,`size`,`file_type_id`,`content_type`,`sha256sum`,`bucket`,`bucket_id`)
         values
         <foreach collection="list" item="item" index="index" separator=",">
-            (#{item.id},#{item.deleted},#{item.createTime},#{item.updateTime},#{item.objectName},#{item.objectId},#{item.filename},#{item.size},#{item.fileTypeId},#{item.contentType},#{item.sha256sum},#{item.bucket})
+            (#{item.id},#{item.deleted},#{item.createTime},#{item.updateTime},#{item.objectName},#{item.objectId},#{item.filename},#{item.size},#{item.fileTypeId},#{item.contentType},#{item.sha256sum},#{item.bucket},#{item.bucket})
         </foreach>
     </insert>