更新, 无 fuck 說

src/main/java/cn/reghao/tnb/file/app/controller/ImageFileController.java

@@ -0,0 +1,52 @@
+package cn.reghao.tnb.file.app.controller;
+
+import cn.reghao.tnb.file.app.db.mapper.FileUrlMapper;
+import cn.reghao.tnb.file.app.model.dto.FileUrlDto;
+import cn.reghao.tnb.file.app.service.FileAccessService;
+import cn.reghao.tnb.file.app.util.LocalStores;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+
+import javax.servlet.http.HttpServletResponse;
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+
+/**
+ * @author reghao
+ * @date 2022-04-24 16:13:10
+ */
+@RestController
+@RequestMapping
+public class ImageFileController {
+    private final FileAccessService fileAccessService;
+    private final FileUrlMapper fileUrlMapper;
+
+    public ImageFileController(FileAccessService fileAccessService, FileUrlMapper fileUrlMapper) {
+        this.fileAccessService = fileAccessService;
+        this.fileUrlMapper = fileUrlMapper;
+    }
+
+    @GetMapping("/image/{filename}")
+    public ResponseEntity<InputStreamResource> image(@PathVariable("filename") String filename) throws Exception {
+        String uploadId = filename.split("\\.")[0];
+        FileUrlDto fileUrlDto = fileUrlMapper.findByUploadId(uploadId);
+        if (fileUrlDto == null) {
+            return ResponseEntity.status(HttpStatus.NOT_FOUND).body(null);
+        }
+        String filePath = LocalStores.getBaseDir(fileUrlDto.getBlockId()) + fileUrlDto.getFilePath();
+
+        FileInputStream fis = new FileInputStream(filePath);
+        InputStreamResource inputStreamResource = new InputStreamResource(fis);
+
+        HttpHeaders httpHeaders = new HttpHeaders();
+        httpHeaders.set("Pragma", "No-cache");
+        httpHeaders.set("Cache-Control", "no-cache");
+        return ResponseEntity.status(HttpStatus.OK).headers(httpHeaders).contentType(MediaType.IMAGE_JPEG)
+                .body(inputStreamResource);
+    }
+}

src/main/java/cn/reghao/tnb/file/app/controller/VideoFileController.java

@@ -2,15 +2,14 @@ package cn.reghao.tnb.file.app.controller;
 
 import cn.reghao.tnb.file.app.db.mapper.FileUrlMapper;
 import cn.reghao.tnb.file.app.model.dto.FileUrlDto;
-import org.springframework.core.io.InputStreamResource;
-import org.springframework.http.HttpHeaders;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
+import cn.reghao.tnb.file.app.service.FileAccessService;
+import cn.reghao.tnb.file.app.util.LocalStores;
 import org.springframework.web.bind.annotation.*;
 
 import javax.servlet.http.HttpServletResponse;
+import java.io.File;
 import java.io.FileInputStream;
+import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
@@ -20,27 +19,28 @@ import java.util.Arrays;
  * @date 2022-04-24 16:13:10
  */
 @RestController
-@RequestMapping
+@RequestMapping("/video")
 public class VideoFileController {
+    private final FileAccessService fileAccessService;
     private final FileUrlMapper fileUrlMapper;
 
-    public VideoFileController(FileUrlMapper fileUrlMapper) {
+    public VideoFileController(FileAccessService fileAccessService, FileUrlMapper fileUrlMapper) {
+        this.fileAccessService = fileAccessService;
         this.fileUrlMapper = fileUrlMapper;
     }
 
-    @GetMapping("/video/{uploadId}")
-    public void videoFile(@PathVariable("uploadId") String uploadId,
-                          @RequestHeader(required = false) String range,
-                          HttpServletResponse response) throws Exception {
-    }
-
-    @GetMapping("/video/playback")
-    public void videoPlayer(@RequestParam("uploadId") String uploadId,
-                            @RequestHeader(required = false) String range,
+    @GetMapping("/playback/{filename}")
+    public void videoPlayer(@PathVariable("filename") String filename, @RequestHeader(required = false) String range,
                             HttpServletResponse response) throws Exception {
-        fileUrlMapper.findByUploadId(uploadId);
+        String uploadId = filename.split("\\.")[0];
+        FileUrlDto fileUrlDto = fileUrlMapper.findByUploadId(uploadId);
+        if (fileUrlDto == null) {
+            response.setStatus(HttpServletResponse.SC_NOT_FOUND);
+            response.getOutputStream().write("".getBytes(StandardCharsets.UTF_8));
+            return;
+        }
+        String filePath = LocalStores.getMountedOn(fileUrlDto.getBlockId()) + "/" + fileUrlDto.getFilePath();
 
-        String filePath = "/home/reghao/Downloads/mp4/test.mp4";
         RandomAccessFile raf = new RandomAccessFile(filePath, "r");
         long fileLength = raf.length();
         long partLength = 1024*1024*2;
@@ -70,17 +70,65 @@ public class VideoFileController {
         response.getOutputStream().write(Arrays.copyOf(bytes, readBytes));
     }
 
-    @GetMapping("/image/{uploadId}")
-    public ResponseEntity<InputStreamResource> videoPlayer(@PathVariable("uploadId") String uploadId) throws Exception {
-        FileUrlDto fileUrlDto = fileUrlMapper.findByUploadId(uploadId);
-        String filePath = fileUrlDto.getPath();
-        FileInputStream fis = new FileInputStream(filePath);
-        InputStreamResource inputStreamResource = new InputStreamResource(fis);
-
-        HttpHeaders httpHeaders = new HttpHeaders();
-        httpHeaders.set("Pragma", "No-cache");
-        httpHeaders.set("Cache-Control", "no-cache");
-        return ResponseEntity.status(HttpStatus.OK).headers(httpHeaders).contentType(MediaType.IMAGE_JPEG)
-                .body(inputStreamResource);
+    @GetMapping("/hls/{uploadId}/{filename:.+}")
+    public void hlsVideoPlayer(@PathVariable("uploadId") String uploadId,
+                               @PathVariable("filename") String filename,
+                               HttpServletResponse response) throws IOException {
+        if (filename.contains("m3u8")) {
+            String localFilePath = "/opt/file/disk/spider/twitter/vid/hls/0/" + filename;
+            FileInputStream fis = new FileInputStream(localFilePath);
+            response.setStatus(HttpServletResponse.SC_OK);
+            response.setContentType("application/x-mpegURL");
+            response.getOutputStream().write(fis.readAllBytes());
+            return;
+        }
+
+        String localFilePath = "/opt/file/disk/spider/twitter/vid/hls/0/" + filename;
+        FileInputStream fis = new FileInputStream(localFilePath);
+
+        response.setContentType("video/mp4");
+        response.setHeader("Content-Length", ""+fis.available());
+        response.setStatus(HttpServletResponse.SC_OK);
+        response.getOutputStream().write(fis.readAllBytes());
+    }
+
+    @GetMapping("/dash/{uploadId}/{filename:.+}")
+    public void dashVideoPlayer(@PathVariable("uploadId") String uploadId,
+                                @PathVariable("filename") String filename,
+                                @RequestHeader(required = false) String range,
+                                HttpServletResponse response) throws IOException {
+        if (filename.contains("mpd")) {
+            String localFilePath = fileAccessService.getLocalFilePath(uploadId);
+            FileInputStream fis = new FileInputStream(localFilePath);
+            response.setStatus(HttpServletResponse.SC_OK);
+            response.setContentType("application/dash+xml; charset=utf-8");
+            response.getOutputStream().write(fis.readAllBytes());
+            return;
+        }
+
+        String localFilePath = fileAccessService.getLocalFilePath(uploadId);
+        String parentDir = new File(localFilePath).getParent();
+        String localFilePath1 = parentDir + File.separator + filename;
+        RandomAccessFile raf = new RandomAccessFile(localFilePath1, "r");
+        long fileLength = raf.length();
+
+        String[] ranges = range.replace("bytes=", "").split(",");
+        String[] firstRange = ranges[0].split("-");
+        long start = Long.parseLong(firstRange[0]);
+        long end = Long.parseLong(firstRange[1]);
+        long expectRead = end-start+1;
+        byte[] bytes = new byte[(int) (expectRead)];
+
+        raf.seek(start);
+        int actualRead = raf.read(bytes);
+        //long end1 = start + actualRead-1;
+        long end1 = Math.min(start+end-1, start+actualRead-1);
+
+        response.setContentType("video/mp4");
+        response.setHeader("Content-Length", ""+actualRead);
+        response.setHeader("Content-Range", "bytes "+start+"-"+end1+"/"+fileLength);
+        response.setHeader("Accept-Ranges", "bytes");
+        response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
+        response.getOutputStream().write(Arrays.copyOf(bytes, actualRead));
     }
 }

src/main/java/cn/reghao/tnb/file/app/model/dto/FileUrlDto.java

@@ -13,6 +13,8 @@ public class FileUrlDto implements Serializable {
     private static final long serialVersionUID = 1L;
 
     private String fileId;
+    private String blockId;
+    private String filePath;
     private String filename;
     private String path;
     private String url;

src/main/java/cn/reghao/tnb/file/app/model/po/FileInfo.java

@@ -33,4 +33,14 @@ public class FileInfo extends BaseObject<Integer> {
         this.size = size;
         this.uploaded = false;
     }
+
+    public FileInfo(String fileId, String sha256sum, String filename, String suffix, long size, String contentType) {
+        this.fileId = fileId;
+        this.sha256sum = sha256sum;
+        this.filename = filename;
+        this.suffix = suffix;
+        this.size = size;
+        this.contentType = contentType;
+        this.uploaded = true;
+    }
 }

src/main/java/cn/reghao/tnb/file/app/model/po/FileUrl.java

@@ -29,4 +29,12 @@ public class FileUrl extends BaseObject<Integer> {
         this.url = url;
         this.path = path;
     }
+
+    public FileUrl(String fileId, String blockId, String filePath, String url, String path) {
+        this.fileId = fileId;
+        this.blockId = blockId;
+        this.filePath = filePath;
+        this.url = url;
+        this.path = path;
+    }
 }

src/main/java/cn/reghao/tnb/file/app/rpc/SpiderFileServiceImpl.java

@@ -0,0 +1,118 @@
+package cn.reghao.tnb.file.app.rpc;
+
+import cn.reghao.jutil.jdk.http.util.UrlFormatter;
+import cn.reghao.jutil.tool.id.IdGenerator;
+import cn.reghao.tnb.file.api.dto.ImageFileDto;
+import cn.reghao.tnb.file.api.dto.VideoFileDto;
+import cn.reghao.tnb.file.api.iface.SpiderFileService;
+import cn.reghao.tnb.file.app.config.DfsProperties;
+import cn.reghao.tnb.file.app.db.mapper.*;
+import cn.reghao.tnb.file.app.model.po.*;
+import cn.reghao.tnb.file.app.util.LocalStores;
+import cn.reghao.tnb.file.app.util.StringUtil;
+import org.apache.dubbo.config.annotation.DubboService;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * @author reghao
+ * @date 2021-12-31 11:26:21
+ */
+@DubboService
+@Service
+public class SpiderFileServiceImpl implements SpiderFileService {
+    private final FileInfoMapper fileInfoMapper;
+    private final FileUrlMapper fileUrlMapper;
+    private final FileUserMapper fileUserMapper;
+    private final VideoFileMapper videoFileMapper;
+    private final ImageFileMapper imageFileMapper;
+    private final IdGenerator idGenerator;
+    private final String domain;
+
+    public SpiderFileServiceImpl(FileInfoMapper fileInfoMapper, FileUrlMapper fileUrlMapper,
+                                 FileUserMapper fileUserMapper, VideoFileMapper videoFileMapper,
+                                 ImageFileMapper imageFileMapper, DfsProperties dfsProperties) {
+        this.fileInfoMapper = fileInfoMapper;
+        this.fileUrlMapper = fileUrlMapper;
+        this.fileUserMapper = fileUserMapper;
+        this.videoFileMapper = videoFileMapper;
+        this.imageFileMapper = imageFileMapper;
+        this.idGenerator = new IdGenerator("upload-id");
+        this.domain = dfsProperties.getDomain();
+    }
+
+    @Transactional(rollbackFor = Exception.class)
+    @Override
+    public String addFile(String fileId, long userId, String url) {
+        FileInfo fileInfo = fileInfoMapper.findByFileId(fileId);
+        if (fileInfo != null) {
+            return fileUrlMapper.findByFileId(fileId).get(0).getUrl();
+        }
+
+        String sha256sum = fileId;
+        String filename = UrlFormatter.getFilename(url);
+        String suffix = StringUtil.getSuffix(filename);
+        long size = 0;
+        String contentType = "";
+        fileInfo = new FileInfo(fileId, sha256sum, filename, suffix, size, contentType);
+
+        String uploadId = idGenerator.getUuid();
+        FileUser fileUser = new FileUser(uploadId, userId, fileId);
+
+        String filePath = url.replace("//file.reghao.cn", "");
+        String blockId = LocalStores.getBlockId(filePath);
+        String path = "";
+        switch (suffix) {
+            case "jpg":
+                path = String.format("/image/%s.%s", uploadId, suffix);
+                break;
+            case "mp4":
+                path = String.format("/video/playback/%s.%s", uploadId, suffix);
+                break;
+            case "m3u8":
+                path = String.format("/video/hls/%s.%s", uploadId, suffix);
+                break;
+            case "mpd":
+                path = String.format("/video/dash/%s/index.%s", uploadId, suffix);
+                break;
+            default:
+        }
+
+        String url1 = String.format("//%s%s", domain, path);
+        FileUrl fileUrl = new FileUrl(fileId, blockId, filePath, url1, path);
+
+        fileInfoMapper.save(fileInfo);
+        fileUrlMapper.save(fileUrl);
+        fileUserMapper.save(fileUser);
+        return url1;
+    }
+
+    @Transactional(rollbackFor = Exception.class)
+    @Override
+    public void addVideoFile(VideoFileDto videoFileDto) {
+        Long userId = videoFileDto.getUserId();
+        String videoFileId = videoFileDto.getVideoId();
+        String videoUrl = videoFileDto.getVideoUrl();
+        String videoUrl1 = addFile(videoFileId, userId, videoUrl);
+
+        String coverFileId = String.format("cover%s", videoFileId);
+        String coverUrl = videoFileDto.getCoverUrl();
+        String coverUrl1 = addFile(coverFileId, userId, coverUrl);
+
+        VideoFile videoFile = new VideoFile(videoFileDto);
+        videoFile.setBaseUrl(videoUrl1);
+        videoFile.setBaseCoverUrl(coverUrl1);
+        videoFileMapper.save(videoFile);
+    }
+
+    @Override
+    public void addImageFiles(List<ImageFileDto> list) {
+        List<ImageFile> list1 = list.stream().map(ImageFile::new).collect(Collectors.toList());
+        if (!list1.isEmpty()) {
+            imageFileMapper.saveAll(list1);
+        }
+    }
+}

src/main/java/cn/reghao/tnb/file/app/rpc/VideoFileServiceImpl.java

@@ -1,80 +0,0 @@
-package cn.reghao.tnb.file.app.rpc;
-
-import cn.reghao.jutil.jdk.http.util.UrlFormatter;
-import cn.reghao.jutil.tool.id.SnowFlake;
-import cn.reghao.tnb.file.api.dto.ImageFileDto;
-import cn.reghao.tnb.file.api.dto.VideoFileDto;
-import cn.reghao.tnb.file.api.iface.VideoFileService;
-import cn.reghao.tnb.file.app.db.mapper.FileInfoMapper;
-import cn.reghao.tnb.file.app.db.mapper.FileUrlMapper;
-import cn.reghao.tnb.file.app.db.mapper.ImageFileMapper;
-import cn.reghao.tnb.file.app.db.mapper.VideoFileMapper;
-import cn.reghao.tnb.file.app.model.po.FileInfo;
-import cn.reghao.tnb.file.app.model.po.FileUrl;
-import cn.reghao.tnb.file.app.model.po.ImageFile;
-import cn.reghao.tnb.file.app.model.po.VideoFile;
-import cn.reghao.tnb.file.app.util.StringUtil;
-import org.apache.dubbo.config.annotation.DubboService;
-import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Transactional;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * @author reghao
- * @date 2021-12-31 11:26:21
- */
-@DubboService
-@Service
-public class VideoFileServiceImpl implements VideoFileService {
-    private final SnowFlake snowFlake;
-    private final FileInfoMapper fileInfoMapper;
-    private final FileUrlMapper fileUrlMapper;
-    private final VideoFileMapper videoFileMapper;
-    private final ImageFileMapper imageFileMapper;
-
-    public VideoFileServiceImpl(FileInfoMapper fileInfoMapper, FileUrlMapper fileUrlMapper,
-                                VideoFileMapper videoFileMapper, ImageFileMapper imageFileMapper) {
-        this.snowFlake = new SnowFlake(1, 1);
-        this.fileInfoMapper = fileInfoMapper;
-        this.fileUrlMapper = fileUrlMapper;
-        this.videoFileMapper = videoFileMapper;
-        this.imageFileMapper = imageFileMapper;
-    }
-
-    @Transactional(rollbackFor = Exception.class)
-    @Override
-    public String addVideoFile(VideoFileDto videoFileDto) {
-        Long userId = videoFileDto.getUserId();
-        String videoId = videoFileDto.getVideoId();
-        String videoUrl = videoFileDto.getVideoUrl();
-
-        FileInfo fileInfo = fileInfoMapper.findByFileId(videoId);
-        if (fileInfo != null) {
-            return fileInfo.getFileId();
-        }
-
-        String filename = UrlFormatter.getFilename(videoUrl);
-        String suffix = StringUtil.getSuffix(filename);
-        String contentType = videoFileDto.getUrlType();
-
-        String sha256sum = String.valueOf(snowFlake.nextId());
-        fileInfo = new FileInfo();
-        FileUrl fileUrl = new FileUrl(videoId, "", videoUrl, "");
-        VideoFile videoFile = new VideoFile(videoFileDto);
-
-        fileInfoMapper.save(fileInfo);
-        fileUrlMapper.save(fileUrl);
-        videoFileMapper.save(videoFile);
-        return fileInfo.getFileId();
-    }
-
-    @Override
-    public void addImageFile(List<ImageFileDto> list) {
-        List<ImageFile> list1 = list.stream().map(ImageFile::new).collect(Collectors.toList());
-        if (!list1.isEmpty()) {
-            imageFileMapper.saveAll(list1);
-        }
-    }
-}

src/main/java/cn/reghao/tnb/file/app/service/FileAccessService.java

@@ -0,0 +1,30 @@
+package cn.reghao.tnb.file.app.service;
+
+import cn.reghao.tnb.file.app.db.mapper.FileUrlMapper;
+import cn.reghao.tnb.file.app.model.dto.FileUrlDto;
+import cn.reghao.tnb.file.app.util.LocalStores;
+import org.springframework.stereotype.Service;
+
+import javax.servlet.http.HttpServletResponse;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * @author reghao
+ * @date 2022-05-30 11:22:48
+ */
+@Service
+public class FileAccessService {
+    private final FileUrlMapper fileUrlMapper;
+
+    public FileAccessService(FileUrlMapper fileUrlMapper) {
+        this.fileUrlMapper = fileUrlMapper;
+    }
+
+    public String getLocalFilePath(String uploadId) {
+        FileUrlDto fileUrlDto = fileUrlMapper.findByUploadId(uploadId);
+        if (fileUrlDto != null) {
+            return LocalStores.getBaseDir(fileUrlDto.getBlockId()) + "/" + fileUrlDto.getFilePath();
+        }
+        return null;
+    }
+}

src/main/java/cn/reghao/tnb/file/app/util/LocalStores.java

@@ -17,7 +17,9 @@ import java.util.*;
  */
 public class LocalStores {
     private static String domain;
+    @Deprecated
     private static final Map<String, LocalStore> storeMap = new HashMap<>();
+    private static final Map<String, LocalStore> storeMap1 = new HashMap<>();
     private static final Map<String, List<String>> subDirs = new HashMap<>();
 
     public static void init(DfsProperties dfsProperties) throws IOException {
@@ -40,11 +42,33 @@ public class LocalStores {
 
             LocalStore localStore = new LocalStore(baseDir, fileStore, blockId, 0.9);
             storeMap.put(baseDir, localStore);
+            storeMap1.put(blockId, localStore);
             subDirs.computeIfAbsent(baseDir, k -> new ArrayList<>());
             createSubDirs(baseDir);
         }
     }
 
+    public static String getDomain() {
+        return domain;
+    }
+
+    // TODO 临时在 VideoFileServiceImpl 中使用
+    @Deprecated
+    public static String getBlockId(String filePath) {
+        List<String> blockIds = new ArrayList<>();
+        for (Map.Entry<String, LocalStore> entry : storeMap1.entrySet()) {
+            String blockId = entry.getKey();
+            LocalStore localStore = entry.getValue();
+            String baseDir = localStore.getBaseDir();
+            File file = new File(baseDir + filePath);
+            if (file.exists()) {
+                blockIds.add(blockId);
+            }
+        }
+
+        return blockIds.get(0);
+    }
+
     private static Map<String, String> getBlockIdMap() throws IOException {
         Map<String, String> map = new HashMap<>();
         File dir = new File("/dev/disk/by-uuid");
@@ -71,6 +95,14 @@ public class LocalStores {
         }
     }
 
+    public static String getMountedOn(String blockId) {
+        return storeMap1.get(blockId).getMountedOn();
+    }
+
+    public static String getBaseDir(String blockId) {
+        return storeMap1.get(blockId).getBaseDir();
+    }
+
     // TODO 优化算法, 处理异常
     public static LocalStore getMaxStore(long size) {
         Map<String, Long> map = new HashMap<>();

src/main/resources/application-dev.yml

@@ -5,15 +5,16 @@ spring:
     port: 6379
     password: Dev@123456
   datasource:
-    url: jdbc:mysql://192.168.0.50:3306/reghao_tnb_tdb?useSSL=false&useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2b8
-    username: test
-    password: Test@123456
+    url: jdbc:mysql://localhost:3306/reghao_tnb_rdb?useSSL=false&useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2b8
+    username: dev
+    password: Dev@123456
 dubbo:
   registry:
     address: zookeeper://localhost:2181
 dfs:
   domain: file.reghao.cn
   baseDirs:
-    - /home/reghao/opt/file/disk0/
-    - /home/reghao/opt/file/disk1/
-    - /home/reghao/opt/file/disk2/
+    - /opt/file/disk/
+    - /home/reghao/opt/file/disk/
+#    - /home/reghao/opt/file/disk1/
+#    - /home/reghao/opt/file/disk2/

src/main/resources/application-dev1.yml

@@ -14,5 +14,6 @@ dubbo:
 dfs:
   domain: file.reghao.cn
   baseDirs:
-    - /home/reghao/opt/file/group0/
-    - /media/reghao/52f23cbd-9a85-4fe4-a184-56110d4bd34e/file/
+    - /home/reghao/opt/file/disk0/
+    - /home/reghao/opt/file/disk1/
+    - /home/reghao/opt/file/disk2/

src/main/resources/application-dev2.yml

@@ -5,21 +5,15 @@ spring:
     port: 6379
     password: Dev@123456
   datasource:
-    url: jdbc:mysql://localhost:3306/reghao_tnb_rdb?useSSL=false&useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2b8
-    username: dev
-    password: Dev@123456
+    url: jdbc:mysql://192.168.0.50:3306/reghao_tnb_tdb?useSSL=false&useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2b8
+    username: test
+    password: Test@123456
 dubbo:
   registry:
     address: zookeeper://localhost:2181
 dfs:
   domain: file.reghao.cn
-  dfsConfigs:
-    - baseDir: /home/reghao/opt/file/group0/
-      group: group0
-      node: node0
-    - baseDir: /home/reghao/opt/file/group1/
-      group: group1
-      node: node0
-    - baseDir: /home/reghao/opt/file/group2/
-      group: group2
-      node: node0
+  baseDirs:
+    - /home/reghao/opt/file/disk0/
+    - /home/reghao/opt/file/disk1/
+    - /home/reghao/opt/file/disk2/

src/main/resources/mapper/FileUrlMapper.xml

@@ -4,9 +4,9 @@
 <mapper namespace="cn.reghao.tnb.file.app.db.mapper.FileUrlMapper">
     <insert id="save" useGeneratedKeys="true" keyProperty="id">
         insert into file_url
-        (`id`,`deleted`,`create_time`,`update_time`,`file_id`,`file_path`,`url`,`path`,`dc_id`,`node_id`)
+        (`id`,`deleted`,`create_time`,`update_time`,`file_id`,`block_id`,`file_path`,`url`,`path`,`dc_id`,`node_id`)
         values 
-        (#{id},#{deleted},#{createTime},#{updateTime},#{fileId},#{filePath},#{url},#{path},#{dcId},#{nodeId})
+        (#{id},#{deleted},#{createTime},#{updateTime},#{fileId},#{blockId},#{filePath},#{url},#{path},#{dcId},#{nodeId})
     </insert>
 
     <update id="updateSetFileUrl">
@@ -18,7 +18,7 @@
         select * from file_url
     </select>
     <select id="findByUploadId" resultType="cn.reghao.tnb.file.app.model.dto.FileUrlDto">
-        select info.id,info.filename,url.path,url.url from file_url url
+        select info.id,info.filename,url.block_id,url.path,url.url,url.file_path from file_url url
         inner join file_user fileUser
         inner join file_info info
         on fileUser.file_id=url.file_id and info.file_id=url.file_id and fileUser.upload_id=#{uploadId}

src/test/java/ConsistentCheckTest.java

@@ -0,0 +1,136 @@
+import cn.reghao.jutil.jdk.security.DigestUtil;
+import cn.reghao.jutil.jdk.text.TextFile;
+import cn.reghao.tnb.file.app.FileApplication;
+import cn.reghao.tnb.file.app.db.mapper.*;
+import cn.reghao.tnb.file.app.model.po.FileInfo;
+import cn.reghao.tnb.file.app.model.po.FileUrl;
+import cn.reghao.tnb.file.app.model.po.VideoFile;
+import cn.reghao.tnb.file.app.util.LocalStores;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import org.apache.tomcat.jni.Local;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.file.*;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.security.NoSuchAlgorithmException;
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * @author reghao
+ * @date 2022-04-23 17:17:50
+ */
+@Slf4j
+@ActiveProfiles("dev")
+@SpringBootTest(classes = FileApplication.class)
+@RunWith(SpringRunner.class)
+public class ConsistentCheckTest {
+    @Autowired
+    FileInfoMapper fileInfoMapper;
+    @Autowired
+    FileUrlMapper fileUrlMapper;
+    @Autowired
+    FileUserMapper fileUserMapper;
+    @Autowired
+    VideoFileMapper videoFileMapper;
+
+    @Test
+    public void test1() throws IOException {
+        /*Map<String, FileUrl> map = fileUrlMapper.findAll().stream()
+                .collect(Collectors.toMap(FileUrl::getFileId, Function.identity()));*/
+        fileInfoMapper.findAll().forEach(fileInfo -> {
+            String fileId = fileInfo.getFileId();
+            if (fileUrlMapper.findByFileId(fileId).isEmpty()) {
+                log.info("{} 在 file_url 中不存在", fileId);
+            }
+        });
+    }
+
+    Map<String, String> map = new HashMap<>();
+    int count = 0;
+    private void walkDir(String dirpath) throws IOException {
+        Path path = Paths.get(dirpath);
+        Files.walkFileTree(path, new FileVisitor<>() {
+            @Override
+            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+                return FileVisitResult.CONTINUE;
+            }
+
+            @Override
+            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                File file1 = file.toFile();
+                if (file1.isFile()) {
+                    count++;
+
+                    String absolutePath = file1.getAbsolutePath();
+                    String name = file1.getName();
+                    String fileId = name;
+                    if (name.contains(".")) {
+                        fileId = name.split("\\.")[0];
+                    }
+
+                    try {
+                        String localSha256sum = DigestUtil.sha256sum(new FileInputStream(file1));
+                        FileInfo fileInfo = fileInfoMapper.findFileInfoBySha256sum(localSha256sum);
+                        if (fileInfo == null) {
+                            log.error("{} 和 file_info 中的 sha256sum 不匹配", fileId);
+                            return FileVisitResult.CONTINUE;
+                        } else {
+                            if (!fileInfo.getFileId().equals(fileId)) {
+                                log.error("{} 和 file_info 中的 fileId 不匹配", fileId);
+                                return FileVisitResult.CONTINUE;
+                            }
+
+                            List<FileUrl> list = fileUrlMapper.findByFileId(fileId);
+                            if (list.size() == 1) {
+                                FileUrl fileUrl = fileUrlMapper.findByFileId(fileId).get(0);
+                                String blockId = fileUrl.getBlockId();
+                                String filePath = fileUrl.getFilePath();
+                                String localFilePath = LocalStores.getMountedOn(blockId) + File.separator + filePath;
+                                if (!localFilePath.equals(absolutePath)) {
+                                    log.error("{} 的 LocalFilePath 不存在", fileId);
+                                    return FileVisitResult.CONTINUE;
+                                }
+                            } else {
+                                log.error("{} 的 FileUrl 不存在", fileId);
+                                return FileVisitResult.CONTINUE;
+                            }
+                        }
+
+                    } catch (NoSuchAlgorithmException e) {
+                        e.printStackTrace();
+                    }
+                }
+                return FileVisitResult.CONTINUE;
+            }
+
+            @Override
+            public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
+                return FileVisitResult.CONTINUE;
+            }
+
+            @Override
+            public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+                return FileVisitResult.CONTINUE;
+            }
+        });
+    }
+
+    @Test
+    public void start() throws IOException {
+        String dirPath = "/home/reghao/opt/file";
+        walkDir(dirPath);
+        log.info("total file = {}", count);
+    }
+}

src/test/java/FileTest.java

@@ -1,112 +0,0 @@
-import cn.reghao.jutil.jdk.security.DigestUtil;
-import cn.reghao.jutil.tool.id.IdGenerator;
-import cn.reghao.tnb.file.app.FileApplication;
-import cn.reghao.tnb.file.app.config.DfsProperties;
-import cn.reghao.tnb.file.app.config.PathUrl;
-import cn.reghao.tnb.file.app.db.mapper.*;
-import cn.reghao.tnb.file.app.model.po.FileInfo;
-import cn.reghao.tnb.file.app.model.po.FileUrl;
-import cn.reghao.tnb.file.app.model.vo.TmpFile;
-import cn.reghao.tnb.file.app.service.FileUrlService;
-import cn.reghao.tnb.file.app.service.TmpFileUrlService;
-import cn.reghao.tnb.file.app.util.LoadBalancer;
-import cn.reghao.tnb.file.app.util.LocalStores;
-import cn.reghao.tnb.file.app.util.StoreDir;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.io.FileUtils;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.test.context.ActiveProfiles;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.nio.file.*;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.security.NoSuchAlgorithmException;
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * @author reghao
- * @date 2022-04-23 17:17:50
- */
-@Slf4j
-@ActiveProfiles("dev")
-@SpringBootTest(classes = FileApplication.class)
-@RunWith(SpringRunner.class)
-public class FileTest {
-    @Autowired
-    FileInfoMapper fileInfoMapper;
-    @Autowired
-    FileUrlMapper fileUrlMapper;
-    @Autowired
-    FileUserMapper fileUserMapper;
-
-    private void walkDir(String dirpath) throws IOException {
-        Path path = Paths.get(dirpath);
-        Files.walkFileTree(path, new FileVisitor<>() {
-            @Override
-            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
-                return FileVisitResult.CONTINUE;
-            }
-
-            @Override
-            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                File file1 = file.toFile();
-                if (file1.isFile()) {
-                    String name = file1.getName();
-                    String fileId = name;
-                    if (name.contains(".")) {
-                        fileId = name.split("\\.")[0];
-                    }
-                    map.put(fileId, file1.getAbsolutePath());
-                }
-
-                return FileVisitResult.CONTINUE;
-            }
-
-            @Override
-            public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
-                return FileVisitResult.CONTINUE;
-            }
-
-            @Override
-            public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
-                return FileVisitResult.CONTINUE;
-            }
-        });
-    }
-
-    @Autowired
-    TmpFileUrlService tmpFileUrlService;
-    Map<String, String> map = new HashMap<>();
-    @Test
-    public void start() throws IOException {
-        String dirPath = "/home/reghao/mnt/hertube/";
-        walkDir(dirPath);
-
-        List<FileInfo> list = fileInfoMapper.findAllByUploaded();
-        list.forEach(fileInfo -> {
-            String fileId = fileInfo.getFileId();
-            List<FileUrl> fileUrls = fileUrlMapper.findByFileId(fileId);
-            if (fileUrls.isEmpty()) {
-                log.error("{} 在 file_url 中不存在", fileId);
-                return;
-            }
-
-            FileUrl fileUrl = fileUrls.get(0);
-            String blockId = fileUrl.getBlockId();
-            if (blockId == null) {
-                String sourcePath = map.get(fileId);
-                TmpFile tmpFile = fileUserMapper.findTmpFileByFileId(fileId);
-                if (tmpFile != null && sourcePath != null) {
-                    tmpFileUrlService.mvFile(tmpFile, sourcePath);
-                }
-            }
-        });
-    }
-}