S3 API 开发中

src/main/java/cn/reghao/dfs/store/config/DfsProperties.java

@@ -22,4 +22,5 @@ public class DfsProperties {
     // TODO 修改为 mountedDirs, 表示磁盘挂载的目录
     private List<String> baseDirs;
     private String encryptKey;
+    private String metaDir;
 }

src/main/java/cn/reghao/dfs/store/meta/RocksClient.java

@@ -0,0 +1,36 @@
+package cn.reghao.dfs.store.meta;
+
+import cn.reghao.dfs.store.config.DfsProperties;
+import cn.reghao.jutil.jdk.serializer.JsonConverter;
+import lombok.extern.slf4j.Slf4j;
+import org.rocksdb.Options;
+import org.rocksdb.RocksDB;
+import org.rocksdb.RocksDBException;
+import org.springframework.stereotype.Component;
+
+import java.nio.charset.StandardCharsets;
+
+/**
+ * @author reghao
+ * @date 2022-11-23 09:33:12
+ */
+@Slf4j
+@Component
+public class RocksClient {
+    private final RocksDB db;
+
+    public RocksClient(DfsProperties dfsProperties) throws RocksDBException {
+        Options options = new Options().setCreateIfMissing(true);
+        this.db = RocksDB.open(options, dfsProperties.getMetaDir());
+    }
+
+    public void put(String key, Object object) throws RocksDBException {
+        String json = JsonConverter.objectToJson(object);
+        db.put(key.getBytes(StandardCharsets.UTF_8), json.getBytes(StandardCharsets.UTF_8));
+    }
+
+    public String get(String key) throws RocksDBException {
+        byte[] value = db.get(key.getBytes(StandardCharsets.UTF_8));
+        return value == null ? null : new String(value);
+    }
+}

src/main/java/cn/reghao/dfs/store/oss/controller/ObjectController.java

@@ -1,15 +1,21 @@
 package cn.reghao.dfs.store.oss.controller;
 
+import cn.reghao.dfs.store.oss.model.FileMeta;
 import cn.reghao.dfs.store.oss.model.object.DeleteObjects;
-import cn.reghao.dfs.store.oss.model.object.PostObject;
+import cn.reghao.dfs.store.oss.service.ObjectService;
 import cn.reghao.jutil.jdk.result.WebBody;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
+import org.rocksdb.RocksDBException;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
 
-import java.util.Map;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 
 /**
  * @author reghao
@@ -19,15 +25,94 @@ import java.util.Map;
 @RestController
 @RequestMapping("/object")
 public class ObjectController {
+    private final ObjectService objectService;
+
+    public ObjectController(ObjectService objectService) {
+        this.objectService = objectService;
+    }
+
+    @ApiOperation("上传对象")
+    @PutMapping(value = "/{objectName:.+}")
+    public String putObject(@PathVariable("objectName") String objectName, MultipartFile file) throws Exception {
+        objectService.putObject(objectName, file);
+        return WebBody.success();
+    }
+
     @ApiOperation("使用 formdata 上传对象")
     @PostMapping(value = "/")
-    public String postObject(@RequestParam("key") String key, @RequestParam("file") MultipartFile file) {
+    public String postObject(@RequestParam("key") String objectName, @RequestParam("file") MultipartFile file) {
         return WebBody.success();
     }
 
-    @ApiOperation("上传对象")
-    @PutMapping(value = "/{objectName}")
-    public String putObject(@PathVariable("objectName") String objectName, MultipartFile file) {
+    @ApiOperation("开启分片上传")
+    @PostMapping(value = "/{objectName}", params = {"uploads"})
+    public String initiateMultipartUpload(@PathVariable("objectName") String objectName) {
+        return WebBody.success();
+    }
+
+    @ApiOperation("分片上传对象")
+    @PutMapping(value = "/{objectName}", params = {"partNumber", "uploadId"})
+    public String uploadPart(@PathVariable("objectName") String objectName,
+                             @RequestParam("partNumber") long partNumber,
+                             @RequestParam("uploadId") String uploadId,
+                             MultipartFile multipartFile) {
+        return WebBody.success();
+    }
+
+    @ApiOperation(value = "完成分片上传")
+    @PostMapping(value = "/{objectName}", params = {"uploadId"})
+    public String completeMultipartUpload(@PathVariable("objectName") String objectName,
+                                          @RequestParam("uploadId") String uploadId) {
+        return WebBody.success();
+    }
+
+    @ApiOperation(value = "终止分片上传")
+    @DeleteMapping(value = "/{objectName}", params = {"uploadId"})
+    public String abortMultipartUpload(@PathVariable("objectName") String objectName,
+                                       @RequestParam("uploadId") String uploadId) {
+        return WebBody.success();
+    }
+
+    @ApiOperation("获取分片 uploadId 下的所有分片")
+    @GetMapping(value = "/{objectName}", params = {"uploadId"})
+    public String listParts(@PathVariable("objectName") String objectName, @RequestParam("uploadId") String uploadId) {
+        return WebBody.success();
+    }
+
+    @ApiOperation("获取对象的元数据")
+    @RequestMapping(value = "/{objectName}", method = RequestMethod.HEAD)
+    public String headObject(@PathVariable("objectName") String objectName) throws RocksDBException {
+        objectService.headObject(objectName);
+        return WebBody.success();
+    }
+
+    @ApiOperation("获取对象")
+    @GetMapping(value = "/{objectName:.+}")
+    public ResponseEntity<InputStreamResource> getObject(@PathVariable("objectName") String objectName) throws RocksDBException, FileNotFoundException {
+        FileMeta fileMeta = objectService.getObject(objectName);
+        if (fileMeta == null) {
+            return ResponseEntity.status(HttpStatus.NOT_FOUND).body(null);
+        }
+
+        String absolutePath = fileMeta.getAbsolutePath();
+        String contentType = fileMeta.getContentType();
+
+        FileInputStream fis = new FileInputStream(absolutePath);
+        InputStreamResource inputStreamResource = new InputStreamResource(fis);
+        return ResponseEntity.status(HttpStatus.OK)
+                .contentType(MediaType.parseMediaType(contentType))
+                .body(inputStreamResource);
+    }
+
+    @ApiOperation(value = "删除对象")
+    @DeleteMapping(value = "/{objectName}")
+    public String deleteObject(@PathVariable("objectName") String objectName) {
+        return WebBody.success();
+    }
+
+    @ApiOperation(value = "删除多个对象")
+    @PostMapping(value = "/", params = {"delete"})
+    public String deleteMultipleObjects(DeleteObjects deleteObjects) {
         return WebBody.success();
     }
 
@@ -55,21 +140,6 @@ public class ObjectController {
         return WebBody.success();
     }
 
-    @ApiOperation("开启分片上传")
-    @PostMapping(value = "/{objectName}", params = {"uploads"})
-    public String initiateMultipartUpload(@PathVariable("objectName") String objectName) {
-        return WebBody.success();
-    }
-
-    @ApiOperation("分片上传对象")
-    @PutMapping(value = "/{objectName}", params = {"partNumber", "uploadId"})
-    public String uploadPart(@PathVariable("objectName") String objectName,
-                             @RequestParam("partNumber") long partNumber,
-                             @RequestParam("uploadId") String uploadId,
-                             MultipartFile multipartFile) {
-        return WebBody.success();
-    }
-
     @ApiOperation("复制已存在的对象作为源数据上传分片")
     @PutMapping(value = "/{objectName}", params = {"partNumber", "uploadId"}, headers = {"x-amz-copy-source"})
     public String uploadPartCopy(@PathVariable("objectName") String objectName,
@@ -79,32 +149,12 @@ public class ObjectController {
         return WebBody.success();
     }
 
-    @ApiOperation(value = "完成分片上传")
-    @PostMapping(value = "/{objectName}", params = {"uploadId"})
-    public String completeMultipartUpload(@PathVariable("objectName") String objectName,
-                                          @RequestParam("uploadId") String uploadId) {
-        return WebBody.success();
-    }
-
     @ApiOperation(value = "对存储类型为 GLACIER 的对象还原临时副本")
     @PostMapping(value = "/{objectName}", params = {"restore"})
     public String postObjectRestore(@PathVariable("objectName") String objectName) {
         return WebBody.success();
     }
 
-    @ApiOperation(value = "终止分片上传")
-    @DeleteMapping(value = "/{objectName}", params = {"uploadId"})
-    public String abortMultipartUpload(@PathVariable("objectName") String objectName,
-                                       @RequestParam("uploadId") String uploadId) {
-        return WebBody.success();
-    }
-
-    @ApiOperation("获取对象")
-    @GetMapping(value = "/{objectName}")
-    public String getObject(@PathVariable("objectName") String objectName) {
-        return WebBody.success();
-    }
-
     @ApiOperation("获取对象的依法保留状态")
     @GetMapping(value = "/{objectKey}", params = {"legal-hold"})
     public String getObjectLegalHold(@PathVariable("objectKey") String objectKey) {
@@ -122,28 +172,4 @@ public class ObjectController {
     public String getObjectTagging(@PathVariable("objectKey") String objectKey) {
         return WebBody.success();
     }
-
-    @ApiOperation("获取对象的元数据")
-    @RequestMapping(value = "/{objectName}", method = RequestMethod.HEAD)
-    public String headObject(@PathVariable("objectName") String objectName) {
-        return WebBody.success();
-    }
-
-    @ApiOperation("获取分片 uploadId 下的所有分片")
-    @GetMapping(value = "/{objectName}", params = {"uploadId"})
-    public String listParts(@PathVariable("objectName") String objectName, @RequestParam("uploadId") String uploadId) {
-        return WebBody.success();
-    }
-
-    @ApiOperation(value = "删除对象")
-    @DeleteMapping(value = "/{objectName}")
-    public String deleteObject(@PathVariable("objectName") String objectName) {
-        return WebBody.success();
-    }
-
-    @ApiOperation(value = "删除多个对象")
-    @PostMapping(value = "/", params = {"delete"})
-    public String deleteMultipleObjects(DeleteObjects deleteObjects) {
-        return WebBody.success();
-    }
 }

src/main/java/cn/reghao/dfs/store/oss/model/FileMeta.java

@@ -0,0 +1,23 @@
+package cn.reghao.dfs.store.oss.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+import java.io.Serializable;
+
+/**
+ * 文件元数据
+ *
+ * @author reghao
+ * @date 2022-11-21 10:53:10
+ */
+@AllArgsConstructor
+@Getter
+public class FileMeta implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+    private String objectName;
+    private long size;
+    private String absolutePath;
+    private String contentType;
+}

src/main/java/cn/reghao/dfs/store/oss/model/object/FileAttr.java

@@ -1,17 +0,0 @@
-package cn.reghao.dfs.store.oss.model.object;
-
-import java.io.Serializable;
-
-/**
- * @author reghao
- * @date 2022-11-21 10:53:10
- */
-public class FileAttr implements Serializable {
-    private static final long serialVersionUID = 1L;
-
-    private String objectId;
-    private String objectName;
-    private String sha256sum;
-    private long size;
-    private String filename;
-}

src/main/java/cn/reghao/dfs/store/oss/service/ObjectService.java

@@ -0,0 +1,122 @@
+package cn.reghao.dfs.store.oss.service;
+
+import cn.reghao.dfs.store.meta.RocksClient;
+import cn.reghao.dfs.store.oss.model.FileMeta;
+import cn.reghao.dfs.store.service.FileStoreService;
+import cn.reghao.dfs.store.service.FileUrlService;
+import cn.reghao.jutil.jdk.security.DigestUtil;
+import cn.reghao.jutil.jdk.serializer.JsonConverter;
+import cn.reghao.jutil.tool.id.IdGenerator;
+import org.rocksdb.RocksDBException;
+import org.springframework.stereotype.Service;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.util.List;
+
+/**
+ * @author reghao
+ * @date 2022-11-23 09:40:18
+ */
+@Service
+public class ObjectService {
+    private final RocksClient rocksClient;
+    private final IdGenerator idGenerator;
+    private final FileUrlService fileUrlService;
+    private final FileStoreService fileStoreService;
+
+    public ObjectService(RocksClient rocksClient, FileUrlService fileUrlService, FileStoreService fileStoreService) {
+        this.rocksClient = rocksClient;
+        this.idGenerator = new IdGenerator(32, "object-id");
+        this.fileUrlService = fileUrlService;
+        this.fileStoreService = fileStoreService;
+    }
+
+    public void putObject(String objectName, MultipartFile multipartFile) throws Exception {
+        String objectKey = String.format("/%s", objectName);
+        String value = rocksClient.get(objectKey);
+        if (value != null) {
+            return;
+        }
+
+        String contentType = multipartFile.getContentType();
+        long size = multipartFile.getSize();
+        InputStream inputStream = multipartFile.getInputStream();
+        String objectId = idGenerator.stringId();
+        String absolutePath = fileUrlService.genFilePath(size, objectId, "");
+        fileStoreService.saveFile(absolutePath, inputStream);
+
+        FileInputStream fis = new FileInputStream(absolutePath);
+        String sha256sum = DigestUtil.sha256sum(fis);
+        //String md5sum = DigestUtil.md5sum(fis);
+        String json = rocksClient.get(sha256sum);
+        if (json != null) {
+            File file = new File(absolutePath);
+            file.delete();
+
+            List list = JsonConverter.jsonToObject(json, List.class);
+            String objectName1 = (String) list.get(0);
+            FileMeta fileMeta = JsonConverter.jsonToObject(rocksClient.get(objectName1), FileMeta.class);
+            String absolutePath1 = fileMeta.getAbsolutePath();
+            String contentType1 = fileMeta.getContentType();
+
+            FileMeta fileMeta1 = new FileMeta(objectName, size, absolutePath1, contentType1);
+            rocksClient.put(objectKey, fileMeta1);
+
+            list.add(objectKey);
+            rocksClient.put(sha256sum, list);
+            return;
+        }
+
+        FileMeta fileMeta = new FileMeta(objectName, size, absolutePath, contentType);
+        rocksClient.put(objectKey, fileMeta);
+        rocksClient.put(sha256sum, List.of(objectKey));
+    }
+
+    public void postObject() {
+    }
+
+    public void initiateMultipartUpload() {
+    }
+
+    public void uploadPart() {
+    }
+
+    public void completeMultipartUpload() {
+    }
+
+    public void abortMultipartUpload() {
+    }
+
+    public void listParts() {
+    }
+
+    public void headObject(String objectName) throws RocksDBException {
+        String objectKey = String.format("/%s", objectName);
+        String value = rocksClient.get(objectKey);
+        if (value == null) {
+            return;
+        }
+
+        FileMeta fileMeta = JsonConverter.jsonToObject(value, FileMeta.class);
+    }
+
+    public FileMeta getObject(String objectName) throws RocksDBException, FileNotFoundException {
+        String objectKey = String.format("/%s", objectName);
+        String value = rocksClient.get(objectKey);
+        if (value == null) {
+            return null;
+        }
+
+        return JsonConverter.jsonToObject(value, FileMeta.class);
+    }
+
+    public void deleteObject() {
+    }
+
+    public void deleteMultipleObjects() {
+    }
+}

src/main/java/cn/reghao/dfs/store/service/FileUrlService.java

@@ -57,6 +57,12 @@ public class FileUrlService {
         return new PathUrl(filePath, url, path);
     }
 
+    public String genFilePath(long size, String objectId, String suffix) {
+        StoreDir storeDir = loadBalancer.getStoreDir(size);
+        String fileDir = storeDir.getAbsoluteDirPath();
+        return String.format("%s/%s.%s", fileDir, objectId, suffix);
+    }
+
     public StoreDir getStoreDir(String sha256sum, long fileSize) {
         //StoreDir storeDir = loadBalancer.getStoreDir(fileSize, sha256sum);
         return loadBalancer.getStoreDir(fileSize);

src/main/resources/application-dev.yml

@@ -8,5 +8,6 @@ dfs:
   group: 0
   node: 0
   baseDirs:
-    - /opt/file/c42796cb-496e-4621-8e11-cb080bdd474d
-  encryptKey: 5JCdi68CulSDu0TqD4jR
+    - /opt/oss/file/c42796cb-496e-4621-8e11-cb080bdd474d/
+  encryptKey: 5JCdi68CulSDu0TqD4jR
+  metaDir: /opt/oss/meta

src/test/java/FileTest.java

@@ -8,6 +8,8 @@ import org.junit.Test;
 import javax.imageio.ImageIO;
 import java.awt.image.BufferedImage;
 import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.Map;
 
 /**
@@ -60,4 +62,11 @@ public class FileTest {
         MediaResolution reqQuality = MediaQuality.getQuality(width, height);
         System.out.println();
     }
+
+    @Test
+    public void test() throws IOException {
+        File file = new File("/home/reghao/Downloads/991792fe.jpeg");
+        String contentType = Files.probeContentType(Path.of(file.getAbsolutePath()));
+        System.out.println();
+    }
 }

src/test/java/RocksDBTest.java

@@ -1,27 +0,0 @@
-import lombok.extern.slf4j.Slf4j;
-import org.rocksdb.Options;
-import org.rocksdb.RocksDB;
-import org.rocksdb.RocksDBException;
-
-import java.nio.charset.StandardCharsets;
-
-/**
- * @author reghao
- * @date 2022-06-20 14:31:14
- */
-@Slf4j
-public class RocksDBTest {
-    static String metaDir = "/opt/file/meta";
-
-    public static void main(String[] args) throws RocksDBException {
-        Options options = new Options().setCreateIfMissing(true);
-        RocksDB db = RocksDB.open(options, metaDir);
-
-        String key = "key1";
-        /*String value = "json value";
-        db.put(key.getBytes(StandardCharsets.UTF_8), value.getBytes(StandardCharsets.UTF_8));*/
-
-        byte[] value1 = db.get(key.getBytes(StandardCharsets.UTF_8));
-        log.info("value -> {}", new String(value1));
-    }
-}