使用 rocksdb 作为数据存储引擎

src/main/java/cn/reghao/dfs/store/meta/RocksClient.java

@@ -24,13 +24,11 @@ public class RocksClient {
         this.db = RocksDB.open(options, dfsProperties.getMetaDir());
     }
 
-    public void put(String key, Object object) throws RocksDBException {
-        String json = JsonConverter.objectToJson(object);
-        db.put(key.getBytes(StandardCharsets.UTF_8), json.getBytes(StandardCharsets.UTF_8));
+    public void put(String key, byte[] bytes) throws RocksDBException {
+        db.put(key.getBytes(StandardCharsets.UTF_8), bytes);
     }
 
-    public String get(String key) throws RocksDBException {
-        byte[] value = db.get(key.getBytes(StandardCharsets.UTF_8));
-        return value == null ? null : new String(value);
+    public byte[] get(String key) throws RocksDBException {
+        return db.get(key.getBytes(StandardCharsets.UTF_8));
     }
 }

src/main/java/cn/reghao/dfs/store/oss/controller/ObjectController.java

@@ -16,6 +16,7 @@ import org.springframework.web.multipart.MultipartFile;
 
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
+import java.io.IOException;
 
 /**
  * @author reghao
@@ -88,20 +89,8 @@ public class ObjectController {
 
     @ApiOperation("获取对象")
     @GetMapping(value = "/{objectName:.+}")
-    public ResponseEntity<InputStreamResource> getObject(@PathVariable("objectName") String objectName) throws RocksDBException, FileNotFoundException {
-        FileMeta fileMeta = objectService.getObject(objectName);
-        if (fileMeta == null) {
-            return ResponseEntity.status(HttpStatus.NOT_FOUND).body(null);
-        }
-
-        String absolutePath = fileMeta.getAbsolutePath();
-        String contentType = fileMeta.getContentType();
-
-        FileInputStream fis = new FileInputStream(absolutePath);
-        InputStreamResource inputStreamResource = new InputStreamResource(fis);
-        return ResponseEntity.status(HttpStatus.OK)
-                .contentType(MediaType.parseMediaType(contentType))
-                .body(inputStreamResource);
+    public void getObject(@PathVariable("objectName") String objectName) throws RocksDBException, IOException {
+        objectService.getObject(objectName);
     }
 
     @ApiOperation(value = "删除对象")

src/main/java/cn/reghao/dfs/store/oss/service/ObjectService.java

@@ -1,20 +1,25 @@
 package cn.reghao.dfs.store.oss.service;
 
 import cn.reghao.dfs.store.meta.RocksClient;
+import cn.reghao.dfs.store.oss.db.mapper.DataBlockMapper;
+import cn.reghao.dfs.store.oss.db.mapper.FileMetaMapper;
+import cn.reghao.dfs.store.oss.model.DataBlock;
+import cn.reghao.dfs.store.oss.model.FileBlock;
 import cn.reghao.dfs.store.oss.model.FileMeta;
 import cn.reghao.dfs.store.service.FileStoreService;
 import cn.reghao.dfs.store.service.FileUrlService;
 import cn.reghao.jutil.jdk.security.DigestUtil;
 import cn.reghao.jutil.jdk.serializer.JsonConverter;
 import cn.reghao.jutil.tool.id.IdGenerator;
+import cn.reghao.jutil.web.ServletUtil;
 import org.rocksdb.RocksDBException;
 import org.springframework.stereotype.Service;
 import org.springframework.web.multipart.MultipartFile;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.InputStream;
+import javax.servlet.http.HttpServletResponse;
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -24,20 +29,22 @@ import java.util.List;
 @Service
 public class ObjectService {
     private final RocksClient rocksClient;
-    private final IdGenerator idGenerator;
-    private final FileUrlService fileUrlService;
-    private final FileStoreService fileStoreService;
+    private final IdGenerator objectIdGenerator;
+    private final IdGenerator blockIdGenerator;
+    private final FileMetaMapper fileMetaMapper;
+    private final DataBlockMapper dataBlockMapper;
 
-    public ObjectService(RocksClient rocksClient, FileUrlService fileUrlService, FileStoreService fileStoreService) {
+    public ObjectService(RocksClient rocksClient, FileMetaMapper fileMetaMapper, DataBlockMapper dataBlockMapper) {
         this.rocksClient = rocksClient;
-        this.idGenerator = new IdGenerator(32, "object-id");
-        this.fileUrlService = fileUrlService;
-        this.fileStoreService = fileStoreService;
+        this.objectIdGenerator = new IdGenerator(32, "object-id");
+        this.blockIdGenerator = new IdGenerator(32, "block-id");
+        this.fileMetaMapper = fileMetaMapper;
+        this.dataBlockMapper = dataBlockMapper;
     }
 
     public void putObject(String objectName, MultipartFile multipartFile) throws Exception {
         String objectKey = String.format("/%s", objectName);
-        String value = rocksClient.get(objectKey);
+        byte[] value = rocksClient.get(objectKey);
         if (value != null) {
             return;
         }
@@ -45,35 +52,23 @@ public class ObjectService {
         String contentType = multipartFile.getContentType();
         long size = multipartFile.getSize();
         InputStream inputStream = multipartFile.getInputStream();
-        String objectId = idGenerator.stringId();
-        String absolutePath = fileUrlService.genFilePath(size, objectId, "");
-        fileStoreService.saveFile(absolutePath, inputStream);
-
-        FileInputStream fis = new FileInputStream(absolutePath);
-        String sha256sum = DigestUtil.sha256sum(fis);
-        //String md5sum = DigestUtil.md5sum(fis);
-        String json = rocksClient.get(sha256sum);
-        if (json != null) {
-            File file = new File(absolutePath);
-            file.delete();
-
-            List list = JsonConverter.jsonToObject(json, List.class);
-            String objectName1 = (String) list.get(0);
-            FileMeta fileMeta = JsonConverter.jsonToObject(rocksClient.get(objectName1), FileMeta.class);
-            String absolutePath1 = fileMeta.getAbsolutePath();
-            String contentType1 = fileMeta.getContentType();
-
-            FileMeta fileMeta1 = new FileMeta(objectName, size, absolutePath1, contentType1);
-            rocksClient.put(objectKey, fileMeta1);
-
-            list.add(objectKey);
-            rocksClient.put(sha256sum, list);
-            return;
+        String objectId = objectIdGenerator.stringId();
+
+        List<DataBlock> list = new ArrayList<>();
+        int len = 1024*1024;
+        byte[] buf = new byte[len];
+        int index = 0;
+        int readLen;
+        while ((readLen = inputStream.read(buf, 0, len)) != -1) {
+            String blockId = blockIdGenerator.stringId();
+            rocksClient.put(blockId, buf);
+            list.add(new DataBlock(objectId, index, blockId));
+            index++;
         }
 
-        FileMeta fileMeta = new FileMeta(objectName, size, absolutePath, contentType);
-        rocksClient.put(objectKey, fileMeta);
-        rocksClient.put(sha256sum, List.of(objectKey));
+        FileMeta fileMeta = new FileMeta(objectName, objectId, size, contentType, "");
+        dataBlockMapper.saveAll(list);
+        fileMetaMapper.save(fileMeta);
     }
 
     public void postObject() {
@@ -95,23 +90,45 @@ public class ObjectService {
     }
 
     public void headObject(String objectName) throws RocksDBException {
-        String objectKey = String.format("/%s", objectName);
-        String value = rocksClient.get(objectKey);
-        if (value == null) {
+    }
+
+    public void getObject(String objectName) throws RocksDBException, IOException {
+        String range = ServletUtil.getRequest().getHeader("range");
+        HttpServletResponse response = ServletUtil.getResponse();
+        FileMeta fileMeta = fileMetaMapper.findByObjectName(objectName);
+        if (fileMeta == null) {
+            response.setStatus(HttpServletResponse.SC_NOT_FOUND);
+            response.getOutputStream().write("".getBytes(StandardCharsets.UTF_8));
             return;
         }
 
-        FileMeta fileMeta = JsonConverter.jsonToObject(value, FileMeta.class);
-    }
 
-    public FileMeta getObject(String objectName) throws RocksDBException, FileNotFoundException {
-        String objectKey = String.format("/%s", objectName);
-        String value = rocksClient.get(objectKey);
-        if (value == null) {
-            return null;
+        String contentType = fileMeta.getContentType();
+        long size = fileMeta.getSize();
+
+        response.setContentType(contentType);
+        if (range == null) {
+            response.setHeader("Content-Length", ""+size);
+            response.setHeader("Accept-Ranges", "bytes");
+            response.setStatus(HttpServletResponse.SC_OK);
+            response.getOutputStream().write("".getBytes(StandardCharsets.UTF_8));
+            return;
+        } else {
+            long start = Long.parseLong(range.substring(range.indexOf("=") + 1, range.indexOf("-")));
         }
 
-        return JsonConverter.jsonToObject(value, FileMeta.class);
+        response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
+        response.setHeader("Accept-Ranges", "bytes");
+        response.setHeader("Content-Length", ""+size);
+        response.setHeader("Content-Range", "bytes "+0+"-"+(size-1)+"/"+size);
+
+        String objectId = fileMeta.getObjectId();
+        List<DataBlock> list = dataBlockMapper.findByObjectId(objectId);
+        for (DataBlock dataBlock : list) {
+            String blockId = dataBlock.getBlockId();
+            byte[] bytes = rocksClient.get(blockId);
+            response.getOutputStream().write(bytes);
+        }
     }
 
     public void deleteObject() {

src/main/resources/application.yml

@@ -9,8 +9,8 @@ server:
 spring:
   servlet:
     multipart:
-      max-request-size: 30MB
-      max-file-size: 20MB
+      max-request-size: 10GB
+      max-file-size: 10GB
   application:
     name: dfs-store
   profiles: