update dfs-store

dfs-store/pom.xml

@@ -110,6 +110,12 @@
             <version>31.1-jre</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.tika</groupId>
+            <artifactId>tika-core</artifactId>
+            <version>2.7.0</version>
+        </dependency>
+
         <dependency>
             <groupId>io.springfox</groupId>
             <artifactId>springfox-swagger2</artifactId>

dfs-store/src/main/java/cn/reghao/dfs/store/controller/ObjectBasicController.java

@@ -5,7 +5,9 @@ import cn.reghao.dfs.store.model.dto.DeleteObjects;
 import cn.reghao.dfs.store.model.vo.PostResponse;
 import cn.reghao.dfs.store.redis.ds.RedisString;
 import cn.reghao.dfs.store.service.ObjectBasicService;
+import cn.reghao.jutil.jdk.converter.ByteHex;
 import cn.reghao.jutil.jdk.result.WebResult;
+import cn.reghao.jutil.jdk.security.Base64Util;
 import cn.reghao.jutil.jdk.security.DigestUtil;
 import cn.reghao.jutil.web.ServletUtil;
 import cn.reghao.oss.common.OssUtil;
@@ -55,29 +57,36 @@ public class ObjectBasicController {
         String date = servletRequest.getHeader("x-amz-date");
 
         FileInputStream fis = new FileInputStream(file);
-        String sha256sum1 = DigestUtil.sha256sum(fis);
+        byte[] md5 = DigestUtil.md5sum(fis.readAllBytes());
+        String md5Hex = ByteHex.bytes2Hex(md5);
+        String md5Base64 = Base64Util.encode(md5);
+        if (!contentMd5.equals(md5Base64)) {
+            return ResponseEntity.status(500).body("md5 不匹配");
+        }
 
+        fis = new FileInputStream(file);
+        String sha256sum1 = DigestUtil.sha256sum(fis);
         if (!sha256sum.equals(sha256sum1)) {
-            return ResponseEntity.status(500).body("md5 不匹配");
+            return ResponseEntity.status(500).body("sha256sum 不匹配");
         }
 
-        String accessKeyId = OssUtil.getAccessKey();
-        if (accessKeyId == null) {
-            return ResponseEntity.status(500).body("签名不正确1");
+        String accessKey = OssUtil.getAccessKey();
+        if (accessKey == null) {
+            return ResponseEntity.status(500).body("accessKeyId 不存在");
         } else {
-            String secretAccessKey = redisString.get(accessKeyId);
-            if (secretAccessKey == null) {
-                return ResponseEntity.status(500).body("签名不正确2");
+            String secretKey = redisString.get(accessKey);
+            if (secretKey == null) {
+                return ResponseEntity.status(500).body("secretKey 不存在");
             }
 
-            boolean matched = OssUtil.matchSignature(secretAccessKey, sha256sum1);
+            boolean matched = OssUtil.matchSignature(secretKey, sha256sum1);
             if (!matched) {
-                return ResponseEntity.status(500).body("签名不正确3");
+                return ResponseEntity.status(500).body("签名不正确");
             }
         }
 
         String objectName = OssUtil.getObjectName();
-        objectBasicService.putObject(objectName, file, contentType, sha256sum);
+        objectBasicService.putObject(objectName, file, sha256sum);
 
         DateFormat df = new SimpleDateFormat("EEE, dd MMM yyyy hh:mm:ss 'GMT'", Locale.CHINA);
         df.setTimeZone(TimeZone.getTimeZone("GMT"));
@@ -88,7 +97,7 @@ public class ObjectBasicController {
                 .header("Content-Length", "1")
                 .header("Connection", "close")
                 .header("Server", "TNBCloudOSS")
-                .eTag(sha256sum1)
+                .eTag(md5Hex)
                 .build();
     }
 
@@ -136,7 +145,8 @@ public class ObjectBasicController {
     @ApiOperation("获取对象")
     @GetMapping(value = "/**")
     public void getObject() throws IOException {
-        String uri = ServletUtil.getRequest().getRequestURI();
+        HttpServletRequest servletRequest = ServletUtil.getRequest();
+        String uri = servletRequest.getRequestURI();
         String uri1 = URLDecoder.decode(uri, StandardCharsets.UTF_8);
 
         String objectName = uri1.replaceFirst("/", "");

dfs-store/src/main/java/cn/reghao/dfs/store/db/mapper/FileMetaMapper.java

@@ -18,7 +18,7 @@ import java.util.List;
 public interface FileMetaMapper extends BaseMapper<FileMeta> {
     void update(@Param("objectId") String objectId, @Param("objectName") String objectName);
 
-    FileMeta findBySha256sum(String sha256sum);
+    List<FileMeta> findBySha256sum(String sha256sum);
     FileMeta findByObjectName(String objectName);
     HeadObjectResult findByObjectName1(String objectName);
     List<FileMeta> findAll0(@Param("bucket") String bucket, @Param("max") Integer max, @Param("regex") String regex);

dfs-store/src/main/java/cn/reghao/dfs/store/service/ObjectBasicService.java

@@ -12,6 +12,7 @@ import cn.reghao.jutil.jdk.security.DigestUtil;
 import cn.reghao.jutil.tool.id.IdGenerator;
 import cn.reghao.jutil.web.ServletUtil;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.tika.Tika;
 import org.springframework.stereotype.Service;
 import org.springframework.util.StringUtils;
 
@@ -38,15 +39,14 @@ public class ObjectBasicService {
     private final IdGenerator blockIdGenerator;
     private final FileMetaMapper fileMetaMapper;
     private final DataBlockMapper dataBlockMapper;
-    // 20MiB
-    private final int blockSize = 1024*1024*20;
     // 10MiB
-    private final long partLength = 1024*1024*10;
+    private final int partLength = 1024*1024*10;
     private final FileStoreService fileStoreService;
     private final FileUrlService fileUrlService;
     private final FileTypeService fileTypeService;
     private final LocalCache localCache;
     private final RedisStringObj redisStringObj;
+    private Tika tika = new Tika();
 
     public ObjectBasicService(FileMetaMapper fileMetaMapper, DataBlockMapper dataBlockMapper,
                               FileStoreService fileStoreService, FileUrlService fileUrlService,
@@ -63,24 +63,37 @@ public class ObjectBasicService {
         this.redisStringObj = redisStringObj;
     }
 
-    public void putObject(String objectName, File file, String contentType, String sha256sum) throws Exception {
+    public void putObject(String objectName, File file, String sha256sum) throws Exception {
         String[] names = objectName.split("/");
         String filename = names[names.length-1];
 
-        /*FileMeta fileMeta2 = fileMetaMapper.findByObjectName(objectName);
-        log.info("暂未实现 PUT 存储");*/
+        String contentType = tika.detect(file);
+        FileMeta fileMeta2 = fileMetaMapper.findByObjectName(objectName);
+        if (fileMeta2 != null) {
+            String sha256sum1 = fileMeta2.getSha256sum();
+            if (sha256sum1.equals(sha256sum)) {
+                log.info("{} 内容没有变化", objectName);
+                return;
+            }
 
-        FileMeta fileMeta = fileMetaMapper.findBySha256sum(sha256sum);
-        if (fileMeta == null) {
+            log.info("暂未实现 PUT 存储");
+            return;
+        }
+
+        List<FileMeta> list = fileMetaMapper.findBySha256sum(sha256sum);
+        if (list.isEmpty()) {
             String objectId = objectIdGenerator.stringId();
             long len = file.length();
-            List<DataBlock> list = store(objectId, len, file);
+            List<DataBlock> blocks = store(objectId, len, file);
 
             int fileTypeId = fileTypeService.getFileType1(contentType);
-            fileMeta = new FileMeta(objectName, objectId, filename, len, fileTypeId, contentType, sha256sum, "tnb", 2);
-            dataBlockMapper.saveAll(list);
+            FileMeta fileMeta = new FileMeta(objectName, objectId, filename, len, fileTypeId, contentType, sha256sum, "tnb", 2);
+            dataBlockMapper.saveAll(blocks);
             fileMetaMapper.save(fileMeta);
         } else {
+            FileMeta fileMeta = list.get(0);
+            List<DataBlock> blocks = dataBlockMapper.findByObjectId(fileMeta.getObjectId());
+
             FileMeta fileMeta1 = new FileMeta(objectName, filename, fileMeta);
             fileMetaMapper.save(fileMeta1);
         }
@@ -94,7 +107,7 @@ public class ObjectBasicService {
         String[] names = objectName.split("/");
         String filename = names[names.length-1];
 
-        FileMeta fileMeta = fileMetaMapper.findBySha256sum(sha256sum);
+        /*FileMeta fileMeta = fileMetaMapper.findBySha256sum(sha256sum);
         if (fileMeta == null) {
             int fileTypeId = fileTypeService.getFileType1(contentType);
             fileMeta = new FileMeta(objectName, objectId, filename, len, fileTypeId, contentType, sha256sum, "tnb", 2);
@@ -103,7 +116,7 @@ public class ObjectBasicService {
         } else {
             FileMeta fileMeta1 = new FileMeta(objectName, filename, fileMeta);
             fileMetaMapper.save(fileMeta1);
-        }
+        }*/
     }
 
     private List<DataBlock> store(String objectId, long len, File file) throws IOException {
@@ -132,9 +145,6 @@ public class ObjectBasicService {
         return list;
     }
 
-    public void postObject() {
-    }
-
     public void headObject(String objectName) throws IOException {
         HttpServletResponse response = ServletUtil.getResponse();
         FileMeta fileMeta = fileMetaMapper.findByObjectName(objectName);
@@ -148,51 +158,44 @@ public class ObjectBasicService {
     }
 
     public void getObject(String objectName) throws IOException {
+        String userAgent = ServletUtil.getRequest().getHeader("user-agent");
+        String host = ServletUtil.getRequest().getHeader("host");
         HttpServletResponse response = ServletUtil.getResponse();
-        Object object = redisStringObj.get(objectName);
-        FileMeta fileMeta;
+
         ObjectMeta objectMeta;
+        Object object = redisStringObj.get(objectName);
         if (object != null) {
-            fileMeta = (FileMeta) object;
             objectMeta = (ObjectMeta) object;
         } else {
-            fileMeta = fileMetaMapper.findByObjectName(objectName);
             objectMeta = fileMetaMapper.findObjectMeta(objectName);
         }
 
-        // FileMeta fileMeta = fileMetaMapper.findByObjectName(objectName);
         if (objectMeta == null) {
             response.setStatus(HttpServletResponse.SC_NOT_FOUND);
-
             OutputStream outputStream = response.getOutputStream();
             outputStream.flush();
             outputStream.close();
             return;
         }
 
-        String objectId = objectMeta.getObjectId();
-        String contentType = objectMeta.getContentType();
-        long len = fileMeta.getSize();
-        if (len < partLength) {
-            writeWholeContent(objectId, contentType, len, objectMeta.getAbsolutePath());
-            return;
-        }
-
+        long len = objectMeta.getSize();
         String range = ServletUtil.getRequest().getHeader("range");
         if (range != null) {
             String rangeStr = StringUtils.trimAllWhitespace(range);
             String[] arr = rangeStr.replace("bytes=", "").split("-");
             long start = Long.parseLong(arr[0]);
             if (arr.length == 1) {
-                writePartialContent(objectId, contentType, start, len, objectMeta.getAbsolutePath());
+                writeContentRange(objectMeta, start, objectMeta.getSize());
             } else {
                 ContentRange contentRange = parseContentRange(range, len);
-                byte[] bytes = range1(objectId, contentRange.getStart(), contentRange.getEnd(), objectMeta.getAbsolutePath());
-                writePartialContent(bytes, contentType, contentRange, len);
+                writeContentRange(objectMeta, contentRange.getStart(), contentRange.getEnd());
             }
         } else {
-            writeAcceptRanges(contentType, len);
-            //writeDownloadContent(objectName, objectId, 0, len);
+            if (userAgent.contains("aws-sdk-java") || host.contains("oss.reghao.cn")) {
+                writeDownloadContent(objectMeta);
+            } else {
+                writeWholeContent(objectMeta);
+            }
         }
     }
 
@@ -227,55 +230,28 @@ public class ObjectBasicService {
         return new ContentRange(start, end);
     }
 
-    private void writePartialContent(byte[] bytes, String contentType, ContentRange contentRange, long len) throws IOException {
+    private void writeContentRange(ObjectMeta objectMeta, long start, long end) throws IOException {
+        long contentLength = objectMeta.getSize();
         HttpServletResponse response = ServletUtil.getResponse();
         response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
-        response.setContentType(contentType);
-        response.setHeader("Content-Length", ""+(contentRange.getEnd()-contentRange.getStart()));
-        response.setHeader("Accept-Ranges", "bytes");
-        response.setHeader("Content-Range", "bytes "+contentRange.getStart()+"-"+contentRange.getEnd()+"/"+len);
-
-        OutputStream outputStream = response.getOutputStream();
-        outputStream.write(bytes);
-        outputStream.flush();
-        outputStream.close();
-    }
-
-    private void writePartialContent(String objectId, String contentType, long start, long len, String filePath) throws IOException {
-        HttpServletResponse response = ServletUtil.getResponse();
-        response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
-        response.setContentType(contentType);
-        response.setHeader("Content-Length", ""+(len-start));
+        response.setContentType(objectMeta.getContentType());
+        response.setHeader("Content-Length", ""+contentLength);
         response.setHeader("Accept-Ranges", "bytes");
-        response.setHeader("Content-Range", "bytes "+start+"-"+(len-1)+"/"+len);
+        response.setHeader("Content-Range", "bytes "+start+"-"+(end-1)+"/"+contentLength);
 
         OutputStream outputStream = response.getOutputStream();
         RandomAccessFile raf;
-        String cachePath = localCache.getCache(objectId);
+        String cachePath = localCache.getCache(objectMeta.getObjectId());
         if (cachePath != null && Files.exists(Path.of(cachePath))) {
             raf = new RandomAccessFile(cachePath, "r");
         } else {
-            /*List<DataBlock> list = dataBlockMapper.findByObjectId(objectId);
-            DataBlock dataBlock = list.get(0);
-            String blockId = dataBlock.getBlockId();
-            String absolutePath = dataBlock.getAbsolutePath();*/
-            //localCache.putCache(objectId, filePath);
-            raf = new RandomAccessFile(filePath, "r");
+            raf = new RandomAccessFile(objectMeta.getAbsolutePath(), "r");
         }
-
-        /*List<DataBlock> list = dataBlockMapper.findByObjectId(objectId);
-        DataBlock dataBlock = list.get(0);
-        String blockId = dataBlock.getBlockId();
-        String absolutePath = dataBlock.getAbsolutePath();
-
-        RandomAccessFile raf = new RandomAccessFile(absolutePath, "r");*/
         raf.seek(start);
 
-        // 10MiB
-        int bufSize = 1024*1024*10;
-        byte[] buf = new byte[bufSize];
+        byte[] buf = new byte[partLength];
         int readLen;
-        while ((readLen = raf.read(buf, 0, bufSize)) != -1) {
+        while ((readLen = raf.read(buf, 0, partLength)) != -1) {
             outputStream.write(buf, 0, readLen);
             outputStream.flush();
         }
@@ -283,38 +259,27 @@ public class ObjectBasicService {
         raf.close();
     }
 
-    /**
-     * 下载内容
-     *
-     * @param
-     * @return
-     * @date 2023-04-30 1:12 AM
-     */
-    private void writeDownloadContent(String objectName, String objectId, long start, long len) {
+    private void writeDownloadContent(ObjectMeta objectMeta) {
         HttpServletResponse response = ServletUtil.getResponse();
-        response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
-        response.setContentType("application/octet-stream");
+        response.setStatus(HttpServletResponse.SC_OK);
+        response.setContentType(objectMeta.getContentType());
+        response.setHeader("Content-Length", ""+objectMeta.getSize());
         /*response.setHeader("Content-Length", ""+(len-start));
         response.setHeader("Accept-Ranges", "bytes");
         response.setHeader("Content-Range", "bytes "+start+"-"+(len-1)+"/"+len);*/
 
         try {
-            response.setHeader("Content-Disposition",
-                    "attachment;filename=" + URLEncoder.encode(objectName, StandardCharsets.UTF_8.toString()));
+            response.setHeader("Content-Disposition", "attachment;filename=" +
+                    URLEncoder.encode(objectMeta.getObjectName(), StandardCharsets.UTF_8.toString()));
 
             OutputStream outputStream = response.getOutputStream();
-            List<DataBlock> list = dataBlockMapper.findByObjectId(objectId);
-            DataBlock dataBlock = list.get(0);
-            String blockId = dataBlock.getBlockId();
-            String absolutePath = dataBlock.getAbsolutePath();
+            String absolutePath = objectMeta.getAbsolutePath();
             RandomAccessFile raf = new RandomAccessFile(absolutePath, "r");
-            raf.seek(start);
+            raf.seek(0);
 
-            // 10MiB
-            int bufSize = 1024*1024*10;
-            byte[] buf = new byte[bufSize];
+            byte[] buf = new byte[partLength];
             int readLen;
-            while ((readLen = raf.read(buf, 0, bufSize)) != -1) {
+            while ((readLen = raf.read(buf, 0, partLength)) != -1) {
                 outputStream.write(buf, 0, readLen);
             }
             outputStream.flush();
@@ -325,25 +290,44 @@ public class ObjectBasicService {
         }
     }
 
-    /**
-     * 返回内容
-     *
-     * @param
-     * @return
-     * @date 2023-04-30 1:12 AM
-     */
-    private void writeWholeContent(String objectId, String contentType, long len, String filePath) throws IOException {
+    private void writeDownloadContentSdk(ObjectMeta objectMeta) {
         HttpServletResponse response = ServletUtil.getResponse();
         response.setStatus(HttpServletResponse.SC_OK);
-        response.setContentType(contentType);
-        //response.setHeader("Content-Length", ""+len);
+        response.setContentType(objectMeta.getContentType());
+        response.setHeader("Content-Length", ""+objectMeta.getSize());
+        /*response.setHeader("Accept-Ranges", "bytes");
+        response.setHeader("Content-Range", "bytes "+start+"-"+(len-1)+"/"+len);*/
+
+        try {
+            response.setHeader("Content-Disposition", "attachment;filename=" +
+                    URLEncoder.encode(objectMeta.getObjectName(), StandardCharsets.UTF_8.toString()));
+
+            OutputStream outputStream = response.getOutputStream();
+            String absolutePath = objectMeta.getAbsolutePath();
+            RandomAccessFile raf = new RandomAccessFile(absolutePath, "r");
+            raf.seek(0);
+
+            byte[] buf = new byte[partLength];
+            int readLen;
+            while ((readLen = raf.read(buf, 0, partLength)) != -1) {
+                outputStream.write(buf, 0, readLen);
+            }
+            outputStream.flush();
+            outputStream.close();
+            raf.close();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    private void writeWholeContent(ObjectMeta objectMeta) throws IOException {
+        HttpServletResponse response = ServletUtil.getResponse();
+        response.setStatus(HttpServletResponse.SC_OK);
+        response.setContentType(objectMeta.getContentType());
+        response.setHeader("Content-Length", ""+objectMeta.getSize());
 
         OutputStream outputStream = response.getOutputStream();
-        /*List<DataBlock> list = dataBlockMapper.findByObjectId(objectId);
-        DataBlock dataBlock = list.get(0);
-        String blockId = dataBlock.getBlockId();
-        String absolutePath = dataBlock.getAbsolutePath();*/
-        RandomAccessFile raf = new RandomAccessFile(filePath, "r");
+        RandomAccessFile raf = new RandomAccessFile(objectMeta.getAbsolutePath(), "r");
 
         // 1MiB
         int bufSize = 1024*1024;
@@ -357,22 +341,7 @@ public class ObjectBasicService {
         raf.close();
     }
 
-    private byte[] range1(String objectId, long start, long end, String filePath) throws IOException {
-        // String absolutePath = dataBlockMapper.findByObjectId(objectId).get(0).getAbsolutePath();
-        RandomAccessFile raf = new RandomAccessFile(filePath, "r");
-        raf.seek(start);
-        long bufSize = end - start;
-        byte[] bytes = new byte[(int) bufSize];
-        int readBytes = raf.read(bytes);
-        raf.close();
-        return bytes;
-    }
-
-    private byte[] mergeByteArray(byte[] arr1, byte[] arr2) {
-        byte[] arr3 = new byte[arr1.length + arr2.length];
-        System.arraycopy(arr1, 0, arr3, 0, arr1.length);
-        System.arraycopy(arr2, 0, arr3, arr1.length, arr2.length);
-        return arr3;
+    private void writeResponse(String absolutePath, long start, long end) {
     }
 
     public void deleteObject(String objectName) {

dfs-store/src/test/java/FileTest.java

@@ -0,0 +1,16 @@
+import lombok.extern.slf4j.Slf4j;
+import org.apache.tika.Tika;
+
+/**
+ * @author reghao
+ * @date 2023-05-09 17:20:16
+ */
+@Slf4j
+public class FileTest {
+    public static void main(String[] args) {
+        String filePath = "/home/reghao/Downloads/public.sql";
+
+        Tika tika = new Tika();
+        log.info(tika.detect(filePath));
+    }
+}