update

src/main/java/cn/reghao/tnb/file/app/db/mapper/FileInfoMapper.java

@@ -7,15 +7,19 @@ import cn.reghao.tnb.file.app.model.po.FileInfo;
 import org.apache.ibatis.annotations.Mapper;
 import org.apache.ibatis.annotations.Param;
 
+import java.util.List;
+
 /**
  * @author reghao
  * @date 2021-11-22 14:30:12
  */
 @Mapper
 public interface FileInfoMapper extends BaseMapper<FileInfo> {
+    void updateSetUploaded1(String fileId);
     void updateSetUploaded(String fileId, int fileType, String contentType);
 
     FileDto findBySha256sum(String sha256sum);
     FileInfo findFileInfoBySha256sum(String sha256sum);
     FileInfo findByFileId(String fileId);
+    List<FileInfo> findAllByUploaded();
 }

src/main/java/cn/reghao/tnb/file/app/db/mapper/FileUserMapper.java

@@ -19,4 +19,5 @@ public interface FileUserMapper extends BaseMapper<FileUser> {
     FileUser findByFileAndUserId(String fileId, long userId);
     FileInfo findFileInfoByUploadId(String uploadId);
     List<TmpFile> findTmpFile();
+    TmpFile findTmpFileByFileId(String fileId);
 }

src/main/java/cn/reghao/tnb/file/app/model/po/FileUrl.java

@@ -16,6 +16,7 @@ import lombok.Setter;
 @Getter
 public class FileUrl extends BaseObject<Integer> {
     private String fileId;
+    private String blockId;
     private String filePath;
     private String url;
     private String path;

src/main/java/cn/reghao/tnb/file/app/service/TmpFileUrlService.java

@@ -10,9 +10,6 @@ import org.apache.commons.io.FileUtils;
 import org.springframework.stereotype.Service;
 
 import java.io.File;
-import java.io.IOException;
-import java.nio.file.*;
-import java.nio.file.attribute.BasicFileAttributes;
 import java.util.List;
 
 /**
@@ -60,7 +57,7 @@ public class TmpFileUrlService {
                 if (url.contains("group")) {
                     filePath1 = url1.replace("//static.reghao.cn/group0/node0", "/root/mnt");
                 } else {
-                    filePath1 = url1.replace("//static.reghao.cn", "/root/mnt");
+                    filePath1 = url1.replace("//static.reghao.cn", "/home/reghao/mnt");
                 }
 
                 File source = new File(filePath1);
@@ -79,24 +76,35 @@ public class TmpFileUrlService {
         log.info("start 执行完成...");
     }
 
-    public void start1() {
-        fileUrlMapper.findAll().forEach(fileUrl -> {
-            String url = fileUrl.getUrl();
-            String filePath;
-            if (url.contains("group")) {
-                filePath = url.replace("//static.reghao.cn/group0/node0", "/root/mnt");
-            } else {
-                filePath = url.replace("//static.reghao.cn", "/root/mnt");
+    public void mvFile(TmpFile tmpFile, String sourcePath) {
+        String sha256sum = tmpFile.getSha256sum();
+        long fileSize = tmpFile.getSize();
+        String uploadId = tmpFile.getUploadId();
+        String suffix = tmpFile.getSuffix();
+        String fileId = tmpFile.getFileId();
+
+        try {
+            PathUrl pathUrl = fileUrlService.genVideoPathAndUrl(sha256sum, uploadId, fileSize, fileId, suffix);
+            String blockId = pathUrl.getBlockId();
+            String filePath = pathUrl.getFilePath();
+            String url = pathUrl.getUrl();
+            String path = pathUrl.getPath();
+
+            List<FileUrl> list1 = fileUrlMapper.findByFileId(fileId);
+            if (list1.isEmpty()) {
+                return;
             }
 
-            File file = new File(filePath);
-            if (!file.exists()) {
-                log.error("{} not exist", filePath);
+            File source = new File(sourcePath);
+            File dest = new File(filePath);
+            if (source.exists()) {
+                FileUtils.copyFile(source, dest, true);
+                fileUrlMapper.updateSetFileUrl(fileId, blockId, filePath);
             } else {
-                log.info("{} exist", filePath);
+                log.error("source file {} not exist", filePath);
             }
-        });
-
-        log.info("start1 执行完成...");
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
     }
 }

src/main/java/cn/reghao/tnb/file/app/util/FileLifecycle.java

@@ -1,14 +1,11 @@
 package cn.reghao.tnb.file.app.util;
 
 import cn.reghao.tnb.file.app.config.DfsProperties;
-import cn.reghao.tnb.file.app.service.TmpFileUrlService;
 import org.springframework.beans.factory.DisposableBean;
 import org.springframework.boot.ApplicationArguments;
 import org.springframework.boot.ApplicationRunner;
 import org.springframework.stereotype.Component;
 
-import java.io.IOException;
-
 /**
  * @author reghao
  * @date 2022-03-23 09:22:01
@@ -16,17 +13,14 @@ import java.io.IOException;
 @Component
 public class FileLifecycle implements ApplicationRunner, DisposableBean {
     private final DfsProperties dfsProperties;
-    private final TmpFileUrlService tmpFileUrlService;
 
-    public FileLifecycle(DfsProperties dfsProperties, TmpFileUrlService tmpFileUrlService) {
+    public FileLifecycle(DfsProperties dfsProperties) {
         this.dfsProperties = dfsProperties;
-        this.tmpFileUrlService = tmpFileUrlService;
     }
 
     @Override
     public void run(ApplicationArguments args) throws Exception {
         LocalStores.init(dfsProperties);
-        tmpFileUrlService.start();
     }
 
     @Override

src/main/java/cn/reghao/tnb/file/app/util/LocalStore.java

@@ -39,6 +39,10 @@ public class LocalStore {
         this.max = result.longValue();
     }
 
+    public void setAvailable(long available) {
+        this.available = available;
+    }
+
     public boolean available(long len) throws IOException {
         return total - fileStore.getUsableSpace() + len > max;
     }

src/main/java/cn/reghao/tnb/file/app/util/LocalStores.java

@@ -72,30 +72,25 @@ public class LocalStores {
     }
 
     // TODO 优化算法, 处理异常
-    public static LocalStore getMaxStore(long size) throws IOException {
+    public static LocalStore getMaxStore(long size) {
         Map<String, Long> map = new HashMap<>();
         for (Map.Entry<String, LocalStore> entry : storeMap.entrySet()) {
-            String storePath = entry.getKey();
+            String baseDir = entry.getKey();
             LocalStore localStore = entry.getValue();
             long availSize = localStore.getAvailable() - size;
-            map.put(storePath, availSize);
+            map.put(baseDir, availSize);
         }
 
-        List<String> result = new ArrayList<>();
-        // 降序排列
+        List<String> baseDirs = new ArrayList<>();
+        // baseDirs 中的元素升序排列
         map.entrySet().stream()
                 .sorted(Map.Entry.comparingByValue())
-                .forEachOrdered(b -> result.add(b.getKey()));
+                .forEachOrdered(b -> baseDirs.add(b.getKey()));
 
-        String store = result.get(0);
-        LocalStore localStore = storeMap.get(store);
-
-
-        if (localStore != null) {
-            return localStore;
-        } else {
-            throw new IOException("");
-        }
+        String baseDir = baseDirs.get(baseDirs.size()-1);
+        LocalStore localStore = storeMap.get(baseDir);
+        localStore.setAvailable(map.get(baseDir));
+        return localStore;
     }
 
     public static List<String> getSubDirs(String store) {

src/main/resources/application-dev.yml

@@ -14,5 +14,6 @@ dubbo:
 dfs:
   domain: file.reghao.cn
   baseDirs:
-    - /home/reghao/opt/file/group0/
-    - /media/reghao/52f23cbd-9a85-4fe4-a184-56110d4bd34e/file/
+    - /home/reghao/opt/file/disk0/
+    - /home/reghao/opt/file/disk1/
+    - /home/reghao/opt/file/disk2/

src/main/resources/mapper/FileInfoMapper.xml

@@ -17,6 +17,10 @@
         </foreach>
     </insert>
 
+    <update id="updateSetUploaded1">
+        update file_info set update_time=now(),uploaded=1
+        where file_id=#{fileId}
+    </update>
     <update id="updateSetUploaded">
         update file_info set update_time=now(),file_type=#{fileType},content_type=#{contentType},uploaded=1
         where file_id=#{fileId}
@@ -37,4 +41,10 @@
     <select id="findAllByPage" resultType="cn.reghao.tnb.file.app.model.po.FileInfo">
         select * from file_info
     </select>
+    <select id="findAll" resultType="cn.reghao.tnb.file.app.model.po.FileInfo">
+        select * from file_info
+    </select>
+    <select id="findAllByUploaded" resultType="cn.reghao.tnb.file.app.model.po.FileInfo">
+        select * from file_info where uploaded is true
+    </select>
 </mapper>

src/main/resources/mapper/FileUserMapper.xml

@@ -31,4 +31,10 @@
         inner join file_info fileInfo
         on fileUser.file_id=fileInfo.file_id
     </select>
+    <select id="findTmpFileByFileId" resultType="cn.reghao.tnb.file.app.model.vo.TmpFile">
+        select fileUser.upload_id,fileInfo.file_id,fileInfo.size,fileInfo.sha256sum,fileInfo.suffix
+        from file_user fileUser
+        inner join file_info fileInfo
+        on fileUser.file_id=fileInfo.file_id and fileInfo.file_id=#{fileId}
+    </select>
 </mapper>

src/test/java/FileTest.java

@@ -12,6 +12,8 @@ import cn.reghao.tnb.file.app.service.TmpFileUrlService;
 import cn.reghao.tnb.file.app.util.LoadBalancer;
 import cn.reghao.tnb.file.app.util.LocalStores;
 import cn.reghao.tnb.file.app.util.StoreDir;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -22,13 +24,17 @@ import org.springframework.test.context.junit4.SpringRunner;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.nio.file.*;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.security.NoSuchAlgorithmException;
 import java.util.*;
+import java.util.stream.Collectors;
 
 /**
  * @author reghao
  * @date 2022-04-23 17:17:50
  */
+@Slf4j
 @ActiveProfiles("dev")
 @SpringBootTest(classes = FileApplication.class)
 @RunWith(SpringRunner.class)
@@ -39,95 +45,68 @@ public class FileTest {
     FileUrlMapper fileUrlMapper;
     @Autowired
     FileUserMapper fileUserMapper;
-    @Autowired
-    VideoFileMapper videoFileMapper;
-    @Autowired
-    ImageFileMapper imageFileMapper;
-
-    IdGenerator idGenerator = new IdGenerator("upload-id");
-    @Test
-    public void test() {
-        int page = 1;
-        int size = 100;
 
-        List<TmpFile> list = fileUserMapper.findTmpFile();
+    private void walkDir(String dirpath) throws IOException {
+        Path path = Paths.get(dirpath);
+        Files.walkFileTree(path, new FileVisitor<>() {
+            @Override
+            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+                return FileVisitResult.CONTINUE;
+            }
 
-        List<FileInfo> list1 = fileInfoMapper.findAllByPage(page, size);
-        list1.forEach(fileInfo -> {
-            String suffix1 = fileInfo.getSuffix().replace(".", "");
-            //fileInfoMapper.updateSetSuffix(fileInfo.getFileId(), suffix1);
-            //System.out.println();
+            @Override
+            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                File file1 = file.toFile();
+                if (file1.isFile()) {
+                    String name = file1.getName();
+                    String fileId = name;
+                    if (name.contains(".")) {
+                        fileId = name.split("\\.")[0];
+                    }
+                    map.put(fileId, file1.getAbsolutePath());
+                }
 
-            /*String fileId = fileInfo.getFileId();
-            List<FileUrl> fileUrls = fileUrlMapper.findByFileId(fileId);
-            FileUrl fileUrl = fileUrls.get(0);
-            String url = fileUrl.getUrl();
+                return FileVisitResult.CONTINUE;
+            }
 
-            String filePath = url.replace("//static.reghao.cn", "");
-            String path = url.replace("//static.reghao.cn", "");
-            String url1 = url.replace("static.reghao.cn", "file.reghao.cn");
+            @Override
+            public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
+                return FileVisitResult.CONTINUE;
+            }
 
-            fileUrl.setFilePath(filePath);
-            fileUrl.setPath(path);
-            fileUrl.setUrl(url1);
-            System.out.println();*/
+            @Override
+            public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+                return FileVisitResult.CONTINUE;
+            }
         });
     }
 
     @Autowired
-    DfsProperties dfsProperties;
-    @Autowired
-    LoadBalancer loadBalancer;
+    TmpFileUrlService tmpFileUrlService;
+    Map<String, String> map = new HashMap<>();
     @Test
-    public void test1() throws IOException, NoSuchAlgorithmException {
-        LocalStores.init(dfsProperties);
+    public void start() throws IOException {
+        String dirPath = "/home/reghao/mnt/hertube/";
+        walkDir(dirPath);
 
-        String dirPath = "/media/reghao/52f23cbd-9a85-4fe4-a184-56110d4bd34e/工具/过人脸教程+控制器（0_0）/0/";
-        File dir = new File(dirPath);
-        for (File file : Objects.requireNonNull(dir.listFiles())) {
-            if (file.isDirectory()) {
-                continue;
+        List<FileInfo> list = fileInfoMapper.findAllByUploaded();
+        list.forEach(fileInfo -> {
+            String fileId = fileInfo.getFileId();
+            List<FileUrl> fileUrls = fileUrlMapper.findByFileId(fileId);
+            if (fileUrls.isEmpty()) {
+                log.error("{} 在 file_url 中不存在", fileId);
+                return;
             }
 
-            long len = file.length();
-            FileInputStream fis = new FileInputStream(file);
-            String sha256sum = DigestUtil.sha256sum(fis);
-            StoreDir storeDir = loadBalancer.getStoreDir(len, sha256sum);
-            System.out.println(storeDir.getFileDir());
-        }
-    }
-
-    @Autowired
-    FileUrlService fileUrlService;
-    @Test
-    public void test2() throws IOException, NoSuchAlgorithmException {
-        int page = 1;
-        int size = 100;
-
-        List<TmpFile> list = fileUserMapper.findTmpFile();
-        for (TmpFile tmpFile : list) {
-            String sha256sum = tmpFile.getSha256sum();
-            long fileSize = tmpFile.getSize();
-            String uploadId = tmpFile.getUploadId();
-            String suffix = tmpFile.getSuffix();
-            String fileId = tmpFile.getFileId();
-
-            if (!suffix.equals("jpg")) {
-                PathUrl pathUrl = fileUrlService.genVideoPathAndUrl(sha256sum, uploadId, fileSize, fileId, suffix);
-                String blockId = pathUrl.getBlockId();
-                String filePath = pathUrl.getFilePath();
-                String url = pathUrl.getUrl();
-                String path = pathUrl.getPath();
-                fileUrlMapper.updateSetFileUrl(fileId, blockId, filePath);
-                System.out.println();
+            FileUrl fileUrl = fileUrls.get(0);
+            String blockId = fileUrl.getBlockId();
+            if (blockId == null) {
+                String sourcePath = map.get(fileId);
+                TmpFile tmpFile = fileUserMapper.findTmpFileByFileId(fileId);
+                if (tmpFile != null && sourcePath != null) {
+                    tmpFileUrlService.mvFile(tmpFile, sourcePath);
+                }
             }
-        }
-    }
-
-    @Autowired
-    TmpFileUrlService tmpFileUrlService;
-    @Test
-    public void test3() {
-        tmpFileUrlService.start1();
+        });
     }
 }

src/test/java/FileTest1.java

@@ -4,8 +4,8 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.RandomAccessFile;
-import java.nio.file.Files;
-import java.nio.file.Paths;
+import java.nio.file.*;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.EnumSet;
 
 import static java.nio.file.StandardOpenOption.*;