update dfs-store

dfs-store/src/main/java/cn/reghao/dfs/store/controller/ObjectUploadController.java

@@ -54,13 +54,9 @@ public class ObjectUploadController {
             String originalFilename = file.getName();
             long size = file.length();
             File savedFile = fileStoreService.saveFile(file.getAbsolutePath(), contentId, size);
-            putObjectService.putObject(pid, objectName, contentId, savedFile, originalFilename, sha256sum);
+            UploadFileRet uploadFileRet = putObjectService.putObject(pid, objectName, contentId, savedFile, originalFilename, sha256sum);
             // TODO PutMessageConverter 中生成的文件需要显式删除
             FileUtils.deleteQuietly(file);
-
-            String uploadId = sha256sum;
-            String url = String.format("https://oss.reghao.cn/%s", objectName);
-            UploadFileRet uploadFileRet = new UploadFileRet(uploadId, url);
             return WebResult.success(uploadFileRet);
         } catch (Exception e) {
             FileUtils.deleteQuietly(file);
@@ -76,17 +72,13 @@ public class ObjectUploadController {
         File savedFile = fileStoreService.saveFile(file.getInputStream(), contentId, size);
         String sha256sum1 = DigestUtil.sha256sum(savedFile.getAbsolutePath());
         if (!sha256sum.equals(sha256sum1)) {
-            return WebResult.failWithMsg("sha256sum 不匹配");
+            return WebResult.failWithMsg("sha256sum not match");
         }
 
         String objectName = ObjectUtil.getObjectName(channelId);
         String pid = putObjectService.getPid(objectName);
         String originalFilename = file.getOriginalFilename();
-        putObjectService.putObject(pid, objectName, contentId, savedFile, originalFilename, sha256sum);
-
-        String uploadId = sha256sum;
-        String url = String.format("https://oss.reghao.cn/%s", objectName);
-        UploadFileRet uploadFileRet = new UploadFileRet(uploadId, url);
+        UploadFileRet uploadFileRet = putObjectService.putObject(pid, objectName, contentId, savedFile, originalFilename, sha256sum);
         return WebResult.success(uploadFileRet);
     }
 }

dfs-store/src/main/java/cn/reghao/dfs/store/db/repository/ObjectRepository.java

@@ -33,7 +33,7 @@ public class ObjectRepository {
         fileMetaMapper.save(fileMeta);
     }
 
-    @CachePut
+    //@CachePut
     public void saveFileMetas(List<FileMeta> list) {
         fileMetaMapper.saveAll(list);
     }

dfs-store/src/main/java/cn/reghao/dfs/store/inerceptor/AccessLogInterceptor.java

@@ -37,14 +37,13 @@ public class AccessLogInterceptor implements HandlerInterceptor {
     @Override
     public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler)
             throws Exception {
-        return true;
-        /*String uri = request.getRequestURI();
+        String uri = request.getRequestURI();
         String method = request.getMethod();
         if (method.equals("OPTIONS")) {
             return true;
         }
 
-        if (method.equals("GET")) {
+        /*if (method.equals("GET")) {
             String objectName = URLDecoder.decode(uri, StandardCharsets.UTF_8).replaceFirst("/", "");
             FileMeta fileMeta = fileMetaMapper.findByObjectName(objectName);
             if (fileMeta == null) {
@@ -56,7 +55,7 @@ public class AccessLogInterceptor implements HandlerInterceptor {
                 response.setStatus(HttpServletResponse.SC_FORBIDDEN);
                 return false;
             }
-        }
+        }*/
 
         String userAgent = request.getHeader("user-agent");
         String ipv4 = request.getRemoteAddr();
@@ -66,7 +65,7 @@ public class AccessLogInterceptor implements HandlerInterceptor {
         long userId = Long.parseLong(request.getHeader("x-user-id"));
         UserPermission userPermission = userQuery.getUserPermission(userId);
         log.info("{}: {} -> {}", userId, method, uri);
-        return false;*/
+        return true;
     }
 
     @Override

dfs-store/src/main/java/cn/reghao/dfs/store/rpc/FileServiceImpl.java

@@ -3,6 +3,7 @@ package cn.reghao.dfs.store.rpc;
 import cn.reghao.dfs.api.dto.DirProp;
 import cn.reghao.dfs.api.dto.FileProp;
 import cn.reghao.dfs.api.iface.FileService;
+import cn.reghao.dfs.store.config.OssProperties;
 import cn.reghao.dfs.store.db.mapper.FileMetaMapper;
 import cn.reghao.dfs.store.model.po.FileMeta;
 import cn.reghao.dfs.store.service.PutObjectService;
@@ -24,10 +25,12 @@ import java.util.UUID;
 public class FileServiceImpl implements FileService {
     private final PutObjectService putObjectService;
     private final FileMetaMapper fileMetaMapper;
+    private final String domain;
 
-    public FileServiceImpl(PutObjectService putObjectService, FileMetaMapper fileMetaMapper) {
+    public FileServiceImpl(PutObjectService putObjectService, FileMetaMapper fileMetaMapper, OssProperties ossProperties) {
         this.fileMetaMapper = fileMetaMapper;
         this.putObjectService = putObjectService;
+        this.domain = ossProperties.getDomain();
     }
 
     @Override
@@ -97,7 +100,7 @@ public class FileServiceImpl implements FileService {
         long size = fileMeta.getSize();
         String contentType = fileMeta.getContentType();
         String sha256sum = fileMeta.getSha256sum();
-        String url = String.format("https://oss.reghao.cn/%s", fileMeta.getObjectName());
+        String url = String.format("https://%s/%s", domain, fileMeta.getObjectName());
         return new FileProp(path, filename, size, contentType, sha256sum, url);
     }
 

dfs-store/src/main/java/cn/reghao/dfs/store/service/FileStoreService.java

@@ -91,22 +91,6 @@ public class FileStoreService {
         }
 
         Files.copy(inputStream, Path.of(absolutePath), StandardCopyOption.REPLACE_EXISTING);
-
-        File parentDir = file.getParentFile();
-        if (!parentDir.exists()) {
-            FileUtils.forceMkdir(parentDir);
-        }
-
-        FileOutputStream fos = new FileOutputStream(file);
-        // 1MiB
-        int len = 1024*1024;
-        byte[] buf = new byte[len];
-        int readLen;
-        while ((readLen = inputStream.read(buf, 0, len)) != -1) {
-            fos.write(buf, 0, readLen);
-        }
-        fos.close();
-        inputStream.close();
         return file;
     }
 
@@ -116,8 +100,6 @@ public class FileStoreService {
             throw new IOException(absolutePath + " exist");
         }
 
-        Files.copy(inputStream, Path.of(absolutePath), StandardCopyOption.REPLACE_EXISTING);
-
         File parentDir = file.getParentFile();
         if (!parentDir.exists()) {
             FileUtils.forceMkdir(parentDir);

dfs-store/src/main/java/cn/reghao/dfs/store/service/GetObjectService.java

@@ -1,5 +1,6 @@
 package cn.reghao.dfs.store.service;
 
+import cn.reghao.dfs.store.config.OssProperties;
 import cn.reghao.dfs.store.db.mapper.FileMetaMapper;
 import cn.reghao.dfs.store.db.repository.ObjectRepository;
 import cn.reghao.dfs.store.model.dto.ContentRange;
@@ -26,10 +27,12 @@ public class GetObjectService {
     // 10MiB
     private final int bufSize = 1024*1024*10;
     private final ObjectRepository objectRepository;
+    private final String domain;
 
-    public GetObjectService(FileMetaMapper fileMetaMapper, ObjectRepository objectRepository) {
+    public GetObjectService(FileMetaMapper fileMetaMapper, ObjectRepository objectRepository, OssProperties ossProperties) {
         this.fileMetaMapper = fileMetaMapper;
         this.objectRepository = objectRepository;
+        this.domain = ossProperties.getDomain();
     }
     
     public void headObject(String objectName) throws IOException {
@@ -79,7 +82,7 @@ public class GetObjectService {
                 writeContentRange(objectMeta, contentRange.getStart(), contentRange.getEnd());
             }
         } else {
-            if (host.contains("oss.reghao.cn")) {
+            if (host.contains(domain)) {
                 writeWholeContent(objectMeta);
             } else {
                 writeDownloadContent(objectMeta);
@@ -145,18 +148,6 @@ public class GetObjectService {
 
         OutputStream outputStream = response.getOutputStream();
         writeResponse(outputStream, objectMeta.getAbsolutePath(), 0, objectMeta.getSize());
-
-        /*String absolutePath = objectMeta.getAbsolutePath();
-        RandomAccessFile raf = new RandomAccessFile(absolutePath, "r");
-        raf.seek(0);
-        byte[] buf = new byte[bufSize];
-        int readLen;
-        while ((readLen = raf.read(buf, 0, bufSize)) != -1) {
-            outputStream.write(buf, 0, readLen);
-        }
-        outputStream.flush();
-        outputStream.close();
-        raf.close();*/
     }
 
     private void writeWholeContent(ObjectMeta objectMeta) throws IOException {
@@ -167,16 +158,6 @@ public class GetObjectService {
 
         OutputStream outputStream = response.getOutputStream();
         writeResponse(outputStream, objectMeta.getAbsolutePath(), 0, objectMeta.getSize());
-
-        /*RandomAccessFile raf = new RandomAccessFile(objectMeta.getAbsolutePath(), "r");
-        byte[] buf = new byte[bufSize];
-        int readLen;
-        while ((readLen = raf.read(buf, 0, bufSize)) != -1) {
-            outputStream.write(buf, 0, readLen);
-        }
-        outputStream.flush();
-        outputStream.close();
-        raf.close();*/
     }
 
     private void writeResponse(OutputStream outputStream, String absolutePath, long start, long end) throws IOException {

dfs-store/src/main/java/cn/reghao/dfs/store/service/ObjectMultipartUploadService.java

@@ -1,5 +1,6 @@
 package cn.reghao.dfs.store.service;
 
+import cn.reghao.dfs.store.config.OssProperties;
 import cn.reghao.dfs.store.db.mapper.FileMetaMapper;
 import cn.reghao.dfs.store.model.dto.PathUrl;
 import cn.reghao.dfs.store.util.ObjectUtil;
@@ -30,12 +31,14 @@ public class ObjectMultipartUploadService {
     private final Map<String, Set<Long>> map = new HashMap<>();
     private final Map<String, PathUrl> pathMap = new HashMap<>();
     private final PutObjectService putObjectService;
+    private final String domain;
 
     public ObjectMultipartUploadService(FileMetaMapper fileMetaMapper, FileStoreService fileStoreService,
-                                        PutObjectService putObjectService) {
+                                        PutObjectService putObjectService, OssProperties ossProperties) {
         this.fileMetaMapper = fileMetaMapper;
         this.fileStoreService = fileStoreService;
         this.putObjectService = putObjectService;
+        this.domain = ossProperties.getDomain();
     }
 
     public synchronized UploadPrepareRet prepareUpload(UploadPrepare uploadPrepare) {
@@ -45,7 +48,7 @@ public class ObjectMultipartUploadService {
             return new UploadPrepareRet("uploadId", PART_SIZE, false, null);
         } else {
             String objectName = fileMeta.getObjectName();
-            String url = String.format("https://oss.reghao.cn/%s", objectName);
+            String url = String.format("https://%s/%s", domain, objectName);
             return new UploadPrepareRet("uploadId", PART_SIZE, false, url);
         }
     }
@@ -73,7 +76,7 @@ public class ObjectMultipartUploadService {
             String objectName = ObjectUtil.getObjectName(uploadFilePart.getChannelId());
             putObjectService.copyObject(objectName, filename, fileMeta);
 
-            String url = String.format("https://oss.reghao.cn/%s", objectName);
+            String url = String.format("https://%s/%s", domain, objectName);
             return new UploadFileRet(sha256sum, url);
         }
 
@@ -114,7 +117,7 @@ public class ObjectMultipartUploadService {
 
             map.remove(sha256sum);
             pathMap.remove(sha256sum);
-            String url = String.format("https://oss.reghao.cn/%s", objectName);
+            String url = String.format("https://%s/%s", domain, objectName);
             return new UploadFileRet(sha256sum, url);
         }
     }

dfs-store/src/main/java/cn/reghao/dfs/store/service/PutObjectService.java

@@ -1,10 +1,12 @@
 package cn.reghao.dfs.store.service;
 
+import cn.reghao.dfs.store.config.OssProperties;
 import cn.reghao.dfs.store.db.repository.ObjectRepository;
 import cn.reghao.dfs.store.model.po.DataBlock;
 import cn.reghao.dfs.store.model.po.FileMeta;
 import cn.reghao.jutil.jdk.security.DigestUtil;
 import cn.reghao.jutil.jdk.shell.Shell;
+import cn.reghao.oss.common.UploadFileRet;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.FileUtils;
 import org.springframework.stereotype.Service;
@@ -23,17 +25,23 @@ import java.util.UUID;
 public class PutObjectService {
     private final ObjectRepository objectRepository;
     private final FileStoreService fileStoreService;
+    private final String domain;
 
-    public PutObjectService(ObjectRepository objectRepository, FileStoreService fileStoreService) {
+    public PutObjectService(ObjectRepository objectRepository, FileStoreService fileStoreService, OssProperties ossProperties) {
         this.objectRepository = objectRepository;
         this.fileStoreService = fileStoreService;
+        this.domain = ossProperties.getDomain();
     }
 
-    public void putObject(String pid, String objectName, String contentId, File savedFile, String originalFilename, String sha256sum) {
+    public UploadFileRet putObject(String pid, String objectName, String contentId, File savedFile, String originalFilename, String sha256sum) {
         FileMeta fileMeta = objectRepository.getBySha256sum(sha256sum);
         if (fileMeta != null) {
             copyObject(objectName, originalFilename, fileMeta);
             FileUtils.deleteQuietly(savedFile);
+
+            String objectId = objectRepository.getByObjectName(objectName).getObjectId();
+            String url = String.format("https://%s/%s", domain, objectName);
+            return new UploadFileRet(objectId, url);
         } else {
             String savedPath = savedFile.getAbsolutePath();
             long size = savedFile.length();
@@ -45,6 +53,9 @@ public class PutObjectService {
             String blockId = UUID.randomUUID().toString();
             List<DataBlock> list = List.of(new DataBlock(contentId, 0, blockId, savedPath));
             objectRepository.saveObject(fileMeta, list);
+
+            String url = String.format("https://%s/%s", domain, objectName);
+            return new UploadFileRet(objectId, url);
         }
     }
 
@@ -80,25 +91,31 @@ public class PutObjectService {
     }
 
     private void addParent(String objectName) {
-        List<String> list = getParent(objectName);
+        List<String> list = getSortedParent(objectName);
         List<FileMeta> fileMetas = new ArrayList<>();
-        list.forEach(parentName -> {
+        for (int i = 0; i < list.size(); i++) {
+            String parentName = list.get(i);
             FileMeta fileMeta = objectRepository.getByObjectName(parentName);
             if (fileMeta == null) {
-                String pid = "0";
-                int index = parentName.lastIndexOf("/");
+                String pid;
+                if (i == 0) {
+                    pid = "0";
+                } else {
+                    pid = objectRepository.getByObjectName(list.get(i-1)).getObjectId();
+                }
                 String objectId = UUID.randomUUID().toString().replace("-", "");
-                String filename = parentName.substring(index);
-                fileMetas.add(new FileMeta(pid, objectId, objectName, filename));
+                String[] names = parentName.split("/");
+                String filename = names[names.length-1];
+                fileMetas.add(new FileMeta(parentName, objectId, filename, pid));
             }
-        });
+        }
 
         if (!fileMetas.isEmpty()) {
             objectRepository.saveFileMetas(fileMetas);
         }
     }
 
-    private List<String> getParent(String objectName) {
+    private List<String> getSortedParent(String objectName) {
         String[] arr = objectName.split("/");
         List<String> list = new ArrayList<>();
         list.add(arr[0] + "/");

dfs-store/src/main/resources/application-dev.yml

@@ -1,17 +1,17 @@
 dubbo:
   registry:
-    address: zookeeper://192.168.0.110:2181
+    address: zookeeper://127.0.0.1:2181
 spring:
   redis:
     database: 0
-    host: 192.168.0.110
+    host: 127.0.0.1
     port: 6379
-    password: Test@123456
+    password: Dev@123456
   datasource:
-    url: jdbc:mysql://192.168.0.110:3306/reghao_oss_tdb?useSSL=false&useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2b8
-    username: test
-    password: Test@123456
+    url: jdbc:mysql://localhost:3306/reghao_oss_rdb?useSSL=false&useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2b8
+    username: dev
+    password: Dev@123456
 oss:
   domain: oss.reghao.cn
   mountedDirs:
-    - /opt/oss/disk/13f654c8-af87-4710-aac9-7aa086c99aec/
+    - /opt/oss/disk/00b989fc-991b-4d4e-959e-9b6e19299b72/