oss-store 大文件分片上传接口实现中

oss-api/src/main/java/cn/reghao/oss/api/rest/UploadedPart.java

@@ -15,8 +15,19 @@ import java.util.List;
 public class UploadedPart implements Serializable {
     private static final long serialVersionUID = 1L;
 
-    private boolean skipUpload;
     private boolean needMerge;
-    private List<String> uploaded;
+    private List<Integer> uploaded;
+    private boolean skipUpload;
+    private String uploadId;
     private String url;
+
+    public UploadedPart(String uploadId, String url) {
+        this.uploadId = uploadId;
+        this.url = url;
+        this.skipUpload = true;
+    }
+
+    public UploadedPart(List<Integer> uploaded) {
+        this.uploaded = uploaded;
+    }
 }

oss-sdk/src/test/java/OssConsoleClientTest.java

@@ -36,7 +36,7 @@ public class OssConsoleClientTest {
 
         UploadedPart uploadedPart = multipartUploadService.get(file);
         if (uploadedPart != null) {
-            List<String> list = uploadedPart.getUploaded();
+            List<Integer> list = uploadedPart.getUploaded();
         }
 
         /*UploadFileRet uploadFileRet = multipartUploadService.upload(file, channelId);

oss-store/src/main/java/cn/reghao/oss/store/controller/ObjectMultipartUploadController.java

@@ -37,10 +37,10 @@ public class ObjectMultipartUploadController {
         this.cache = cache;
     }
 
-    @ApiOperation(value = "创建对象分片上传")
-    @PostMapping(value = "/", params = {"create"}, produces = MediaType.APPLICATION_JSON_VALUE)
-    public ResponseEntity<String> create(@Validated UploadPrepare uploadPrepare) throws Exception {
-        int channelId = uploadPrepare.getChannelId();
+    @ApiOperation(value = "获取已上传的对象分片")
+    @GetMapping(value = "/", params = {"multiparts"}, produces = MediaType.APPLICATION_JSON_VALUE)
+    public ResponseEntity<String> getUploadedPart(UploadFilePart uploadFilePart) throws Exception {
+        int channelId = uploadFilePart.getChannelId();
         /* permission check */
         String token = ServletUtil.getBearerToken();
         if (token == null) {
@@ -71,34 +71,12 @@ public class ObjectMultipartUploadController {
         }
         AuthContext context = new AuthContext(loginUser);
 
-        UploadPrepareRet uploadPrepareRet = objectMultipartUploadService.prepareUpload(uploadPrepare, objectChannel);
-        return ResponseEntity.status(HttpStatus.OK).body(WebResult.success(uploadPrepareRet));
-    }
-
-    @ApiOperation(value = "获取已上传的对象分片")
-    @PostMapping(value = "/", params = {"get_multipart"}, produces = MediaType.APPLICATION_JSON_VALUE)
-    public ResponseEntity<String> getUploadedPart(String sha256sum) {
-        /* permission check */
-        String token = ServletUtil.getBearerToken();
-        if (token == null) {
-            return ResponseEntity.status(HttpStatus.UNAUTHORIZED)
-                    .body(WebResult.failWithMsg("no token in request"));
-        }
-
-        String secretKey = cache.getIfPresent(token);
-        OssPayload ossPayload = JwtUtil.getOssPayload(token, secretKey);
-        String action = ossPayload.getAction();
-        if (!"upload".equals(action)) {
-            return ResponseEntity.status(HttpStatus.FORBIDDEN)
-                    .body(WebResult.failWithMsg("it's not upload token"));
-        }
-
-        UploadedPart uploadedPart = objectMultipartUploadService.getUploadedPart(sha256sum);
+        UploadedPart uploadedPart = objectMultipartUploadService.getUploadedPart(uploadFilePart, objectChannel);
         return ResponseEntity.status(HttpStatus.OK).body(WebResult.success(uploadedPart));
     }
 
     @ApiOperation(value = "上传对象分片")
-    @PostMapping(value = "/", params = {"multipart"}, produces = MediaType.APPLICATION_JSON_VALUE)
+    @PostMapping(value = "/", params = {"multiparts"}, produces = MediaType.APPLICATION_JSON_VALUE)
     public ResponseEntity<String> uploadPart(MultipartFile file, @Validated UploadFilePart uploadFilePart) throws Exception {
         int channelId = uploadFilePart.getChannelId();
         /* permission check */
@@ -134,10 +112,4 @@ public class ObjectMultipartUploadController {
         UploadFileRet uploadFileRet = objectMultipartUploadService.putFilePart(file.getInputStream(), uploadFilePart, objectChannel);
         return ResponseEntity.status(HttpStatus.OK).body(WebResult.success(uploadFileRet));
     }
-
-    @ApiOperation(value = "合并对象分片")
-    @PostMapping(value = "/", params = {"merge"}, produces = MediaType.APPLICATION_JSON_VALUE)
-    public String merge() {
-        return WebResult.success();
-    }
 }

oss-store/src/main/java/cn/reghao/oss/store/service/FileStoreService.java

@@ -46,7 +46,7 @@ public class FileStoreService {
         }
         LocalStores.init(storeDirs);
 
-        /*for (String diskDir : diskDirs) {
+        /*for (String diskDir : storeDirs) {
             Map<String, Integer> map = new HashMap<>();
             List<SubDirCount> list = dataBlockMapper.findSubDirCount("");
             list.forEach(subDirCount -> {

oss-store/src/main/java/cn/reghao/oss/store/service/ObjectMultipartUploadService.java

@@ -8,7 +8,7 @@ import cn.reghao.oss.store.db.repository.ObjectRepository;
 import cn.reghao.oss.store.model.po.FilePart;
 import cn.reghao.oss.store.model.po.FileMultipart;
 import cn.reghao.oss.store.model.vo.ObjectProp;
-import cn.reghao.oss.store.util.FileType;
+import cn.reghao.oss.store.model.vo.ObjectResult;
 import cn.reghao.oss.store.util.StringUtil;
 import cn.reghao.jutil.jdk.security.DigestUtil;
 import cn.reghao.oss.store.model.po.FileMeta;
@@ -18,6 +18,7 @@ import org.springframework.transaction.annotation.Transactional;
 
 import java.io.*;
 import java.util.*;
+import java.util.stream.Collectors;
 
 /**
  * @author reghao
@@ -44,54 +45,34 @@ public class ObjectMultipartUploadService {
         this.filePartRepository = filePartRepository;
     }
 
-    public synchronized UploadPrepareRet prepareUpload(UploadPrepare uploadPrepare, ObjectChannel objectChannel) throws Exception {
-        String filename = uploadPrepare.getFilename();
-        String sha256sum = uploadPrepare.getSha256sum();
-        String contentType = uploadPrepare.getContentType();
-        long size = uploadPrepare.getSize();
+    public UploadedPart getUploadedPart(UploadFilePart uploadFilePart, ObjectChannel objectChannel) throws Exception {
+        String sha256sum = uploadFilePart.getIdentifier();
         FileMeta fileMeta = objectRepository.getBySha256sum(sha256sum);
-        if (fileMeta == null) {
-            String suffix = StringUtil.getSuffix(filename);
-            ObjectProp objectProp = objectNameService.getObjectProp(objectChannel, suffix);
-            String pid = objectProp.getPid();
-            String objectName = objectProp.getObjectName();
-            String objectId = UUID.randomUUID().toString().replace("-", "");
-            int fileType = FileType.getFileType(contentType);
-            //int scope = objectProp.getScope();
-            String contentId = UUID.randomUUID().toString().replace("-", "");
-
-            int scope = 0;
-            fileMeta = new FileMeta(objectName, objectId, contentId, filename, size, fileType, contentType, sha256sum, pid, scope);
-            objectRepository.saveFileMeta(fileMeta);
-            return new UploadPrepareRet(objectId, PART_SIZE, false, null);
-        } else {
-            String suffix = StringUtil.getSuffix(filename);
+        if (fileMeta != null) {
+            String filename = uploadFilePart.getFilename();
+            String objectName = fileMeta.getObjectName();
+            String suffix = StringUtil.getSuffix(objectName);
+
             ObjectProp objectProp = objectNameService.getObjectProp(objectChannel, suffix);
-            String pid = objectProp.getPid();
-            String toObjectName = objectProp.getObjectName();
-            String toObjectId = UUID.randomUUID().toString().replace("-", "");
-            int scope = objectProp.getScope();
-
-            FileMeta fileMeta1 = new FileMeta(toObjectName, toObjectId, filename, fileMeta, pid, scope);
-            objectRepository.saveFileMeta(fileMeta1);
-            return new UploadPrepareRet(toObjectId, PART_SIZE, true, null);
+            ObjectResult objectResult = putObjectService.copyObject(objectProp, filename, fileMeta);
+
+            String objectId = objectResult.getObjectId();
+            String objectName1 = objectResult.getObjectName();
+            String host = ServletUtil.getHeader("host");
+            String url = String.format("//%s/%s", host, objectName1);
+            return new UploadedPart(objectId, url);
         }
-    }
 
-    public UploadedPart getUploadedPart(String sha256sum) {
         FileMultipart fileMultipart = filePartRepository.getFileMultipart(sha256sum);
         if (fileMultipart == null) {
             return null;
         }
 
         String contentId = fileMultipart.getContentId();
-        List<FilePart> list = filePartRepository.getFileParts(contentId);
-        UploadedPart uploadedPart = new UploadedPart();
-        uploadedPart.setSkipUpload(false);
-        uploadedPart.setNeedMerge(false);
-        uploadedPart.setUrl("");
-        uploadedPart.setUploaded(Collections.emptyList());
-        return uploadedPart;
+        List<Integer> list = filePartRepository.getFileParts(contentId).stream()
+                .map(FilePart::getChunkNumber)
+                .collect(Collectors.toList());
+        return new UploadedPart(list);
     }
 
     /**
@@ -119,23 +100,27 @@ public class ObjectMultipartUploadService {
             String suffix = StringUtil.getSuffix(objectName);
 
             ObjectProp objectProp = objectNameService.getObjectProp(objectChannel, suffix);
-            putObjectService.copyObject(objectProp, filename, fileMeta);
+            ObjectResult objectResult = putObjectService.copyObject(objectProp, filename, fileMeta);
 
+            String objectId = objectResult.getObjectId();
+            String objectName1 = objectResult.getObjectName();
             String host = ServletUtil.getHeader("host");
-            String url = String.format("//%s/%s", host, objectProp);
-            return new UploadFileRet(sha256sum, url);
+            String url = String.format("//%s/%s", host, objectName1);
+            return new UploadFileRet(objectId, url);
         }
 
         List<FilePart> list = new ArrayList<>();
-        String contentId = filePartRepository.getFileMultipart(sha256sum).getContentId();
-        if (contentId == null) {
+        String contentId;
+        FileMultipart fileMultipart = filePartRepository.getFileMultipart(sha256sum);
+        if (fileMultipart == null) {
             contentId = UUID.randomUUID().toString().replace("-", "");
             String absolutePath = fileStoreService.genFilePath(contentId, totalSize, "");
             fileStoreService.createSparseFile(absolutePath, totalSize);
 
-            FileMultipart fileMultipart = new FileMultipart(contentId, uploadFilePart, absolutePath);
+            fileMultipart = new FileMultipart(contentId, uploadFilePart, absolutePath);
             filePartRepository.saveFileParts(fileMultipart);
         } else {
+            contentId = fileMultipart.getContentId();
             list = filePartRepository.getFileParts(contentId);
         }
 
@@ -170,7 +155,7 @@ public class ObjectMultipartUploadService {
 
         filePartRepository.updateSetUploaded(sha256sum);
         String host = ServletUtil.getHeader("host");
-        String url = String.format("//%s/%s", host, objectProp);
+        String url = String.format("//%s/%s", host, objectProp.getObjectName());
         return new UploadFileRet(sha256sum, url);
     }
 }