更新 cn.reghao.oss.store.util.store 模块

oss-store/bin/application-dev.yml

@@ -5,16 +5,11 @@ dubbo:
   registry:
     address: zookeeper://127.0.0.1:2181
 spring:
-  redis:
-    database: 0
-    host: 127.0.0.1
-    port: 6379
-    password: Dev@123456
   datasource:
     url: jdbc:mysql://localhost:3306/reghao_oss_rdb?useSSL=false&useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2b8
     username: dev
     password: Dev@123456
 oss:
   domain: oss.reghao.cn
-  mountedDirs:
+  diskDirs:
     - /opt/oss/disk/13f654c8-af87-4710-aac9-7aa086c99aec/

oss-store/bin/application-test.yml

@@ -5,16 +5,11 @@ dubbo:
   registry:
     address: zookeeper://192.168.0.110:2181
 spring:
-  redis:
-    database: 0
-    host: 192.168.0.110
-    port: 6379
-    password: Test@123456
   datasource:
     url: jdbc:mysql://192.168.0.110:3306/reghao_oss_tdb?useSSL=false&useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2b8
     username: test
     password: Test@123456
 oss:
   domain: oss.reghao.cn
-  mountedDirs:
+  diskDirs:
     - /opt/oss/disk/13f654c8-af87-4710-aac9-7aa086c99aec/

oss-store/src/main/java/cn/reghao/oss/store/config/CacheConfig.java

@@ -19,22 +19,6 @@ import java.util.concurrent.TimeUnit;
 @EnableCaching
 @Configuration
 public class CacheConfig {
-    /*@Bean
-    public CacheManager cacheManager(RedisTemplate<String, Object> template) {
-        RedisCacheConfiguration config = RedisCacheConfiguration
-                .defaultCacheConfig()
-                .serializeKeysWith(RedisSerializationContext.SerializationPair.fromSerializer(template.getStringSerializer()))
-                .serializeValuesWith(RedisSerializationContext.SerializationPair.fromSerializer(template.getValueSerializer()))
-                .disableCachingNullValues()
-                .entryTtl(Duration.ofHours(1));
-
-        return RedisCacheManager.RedisCacheManagerBuilder
-                        .fromConnectionFactory(Objects.requireNonNull(template.getConnectionFactory()))
-                        .cacheDefaults(config)
-                        .transactionAware()
-                        .build();
-    }*/
-
     @Bean
     public CacheManager cacheManager() {
         CaffeineCacheManager cacheManager = new CaffeineCacheManager();

oss-store/src/main/java/cn/reghao/oss/store/config/OssProperties.java

@@ -17,5 +17,5 @@ import java.util.List;
 @ConfigurationProperties(prefix = "oss")
 public class OssProperties {
     private String domain;
-    private List<String> mountedDirs;
+    private List<String> diskDirs;
 }

oss-store/src/main/java/cn/reghao/oss/store/config/spring/FileLifecycle.java

@@ -1,6 +1,8 @@
 package cn.reghao.oss.store.config.spring;
 
 import cn.reghao.oss.store.config.OssProperties;
+import cn.reghao.oss.store.db.mapper.DataBlockMapper;
+import cn.reghao.oss.store.model.vo.SubDirCount;
 import cn.reghao.oss.store.task.FileTask;
 import cn.reghao.oss.store.util.store.LocalStores;
 import lombok.extern.slf4j.Slf4j;
@@ -9,6 +11,11 @@ import org.springframework.boot.ApplicationArguments;
 import org.springframework.boot.ApplicationRunner;
 import org.springframework.stereotype.Component;
 
+import java.io.File;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 /**
  * @author reghao
  * @date 2022-03-23 09:22:01
@@ -17,18 +24,36 @@ import org.springframework.stereotype.Component;
 @Component
 public class FileLifecycle implements ApplicationRunner, DisposableBean {
     private final OssProperties ossProperties;
+    private final DataBlockMapper dataBlockMapper;
     private final FileTask fileTask;
 
-    public FileLifecycle(OssProperties ossProperties, FileTask fileTask) {
+    public FileLifecycle(OssProperties ossProperties, DataBlockMapper dataBlockMapper, FileTask fileTask) {
         this.ossProperties = ossProperties;
+        this.dataBlockMapper = dataBlockMapper;
         this.fileTask = fileTask;
     }
 
     @Override
     public void run(ApplicationArguments args) throws Exception {
-        log.info("加载本地磁盘数据...");
-        LocalStores.init(ossProperties);
-        log.info("本地磁盘数据加载完成...");
+        log.info("初始化本地磁盘...");
+        List<String> diskDirs = ossProperties.getDiskDirs();
+        LocalStores.init(diskDirs);
+
+        for (String diskDir : diskDirs) {
+            List<SubDirCount> list = dataBlockMapper.findSubDirCount("");
+            Map<String, Integer> map = new HashMap<>();
+            list.forEach(subDirCount -> {
+                String relativeDir = subDirCount.getRelativeDir();
+                String storeDir = diskDir + relativeDir;
+                File file = new File(storeDir);
+
+                int total = subDirCount.getTotal();
+                map.put(file.getAbsolutePath(), total);
+            });
+
+            LocalStores.initStoreDirs(diskDir, map);
+        }
+        log.info("本地磁盘数据初始化完成...");
 
         log.info("执行文件任务...");
         fileTask.exec();

oss-store/src/main/java/cn/reghao/oss/store/service/FileStoreService.java

@@ -22,13 +22,13 @@ import static java.nio.file.StandardOpenOption.*;
 public class FileStoreService {
     private final LoadBalancer loadBalancer;
 
-    public FileStoreService(LoadBalancer loadBalancer) {
-        this.loadBalancer = loadBalancer;
+    public FileStoreService() {
+        this.loadBalancer = new LoadBalancer();
     }
 
     public String genFilePath(String contentId, long size, String suffix) {
         StoreDir storeDir = loadBalancer.getStoreDir(size);
-        String fileDir = storeDir.getAbsoluteDirPath();
+        String fileDir = storeDir.getBaseDir();
         return String.format("%s/%s%s", fileDir, contentId, suffix);
     }
 
@@ -94,7 +94,6 @@ public class FileStoreService {
         return file;
     }
 
-    @Deprecated
     public void saveFile(InputStream inputStream, String absolutePath) throws IOException {
         File file = new File(absolutePath);
         if (file.exists()) {

oss-store/src/main/java/cn/reghao/oss/store/util/store/LoadBalancer.java

@@ -1,19 +1,16 @@
 package cn.reghao.oss.store.util.store;
 
-import org.springframework.stereotype.Component;
-
 import java.util.*;
 
 /**
  * @author reghao
  * @date 2022-03-22 13:53:08
  */
-@Component
 public class LoadBalancer {
     public StoreDir getStoreDir(long fileSize) {
         LocalStore localStore = LocalStores.getMaxStore(fileSize);
-        List<StoreDir> subDirs = LocalStores.getSubDirs(localStore.getMountedOn());
-        subDirs.sort(Comparator.comparingInt(StoreDir::getCount));
+        List<StoreDir> subDirs = LocalStores.getSubDirs(localStore.getDiskDir());
+        subDirs.sort(Comparator.comparingInt(StoreDir::getTotal));
         return subDirs.get(0);
     }
 }

oss-store/src/main/java/cn/reghao/oss/store/util/store/LocalStore.java

@@ -1,55 +1,52 @@
 package cn.reghao.oss.store.util.store;
 
-import lombok.Getter;
-
-import java.io.IOException;
 import java.math.BigDecimal;
 import java.math.RoundingMode;
-import java.nio.file.FileStore;
 import java.util.concurrent.atomic.AtomicLong;
 
 /**
+ * 表示一块磁盘
+ *
  * @author reghao
  * @date 2022-05-23 15:00:59
  */
-@Getter
 public class LocalStore {
-    private final String baseDir;
-    private final String fs;
-    private final String fsType;
-    private final String mountedOn;
+    // disk mounted directory
+    private final String diskDir;
     private final long total;
     private final AtomicLong available;
     private final long max;
 
-    public LocalStore(String baseDir, FileStore fileStore, double maxPercent) throws IOException {
-        this.baseDir = baseDir;
-        this.fs = fileStore.name();
-        this.fsType = fileStore.type();
-        this.mountedOn = fileStore.toString().replace(fs, "").replace(" ()", "");
-        this.total = fileStore.getTotalSpace();
-        this.available = new AtomicLong(fileStore.getUsableSpace());
-
+    public LocalStore(String diskDir, long total, long available, double maxPercent) {
+        this.diskDir = diskDir;
+        this.total = total;
+        this.available = new AtomicLong(available);
         BigDecimal bigDecimal1 = new BigDecimal(total*10);
         BigDecimal bigDecimal2 = new BigDecimal(maxPercent*10);
         BigDecimal result = bigDecimal1.divide(bigDecimal2, RoundingMode.DOWN);
         this.max = result.longValue();
     }
 
-    public void setAvail(long avail) {
-        available.getAndSet(avail);
+    public String getDiskDir() {
+        return diskDir;
+    }
+
+    public long getTotal() {
+        return total;
     }
 
-    public long getAvail() {
+    public long getAvailable() {
         return available.get();
     }
 
+    public void setCurrentAvailable(long currentAvailable) {
+        available.getAndSet(currentAvailable);
+    }
+
     @Override
     public int hashCode() {
         int result = 17;
-        result = result * 31 + fs.hashCode();
-        result = result * 31 + fsType.hashCode();
-        result = result * 31 + mountedOn.hashCode();
+        result = result * 31 + diskDir.hashCode();
         return result;
     }
 
@@ -61,7 +58,7 @@ public class LocalStore {
 
         if (other instanceof LocalStore) {
             LocalStore o = (LocalStore) other;
-            return o.fs.equals(fs) && o.fsType.equals(fsType) && o.mountedOn.equals(mountedOn);
+            return o.diskDir.equals(diskDir);
         } else {
             return false;
         }

oss-store/src/main/java/cn/reghao/oss/store/util/store/LocalStores.java

@@ -1,15 +1,11 @@
 package cn.reghao.oss.store.util.store;
 
-import cn.reghao.oss.store.config.OssProperties;
-import org.apache.commons.io.FileUtils;
-
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.FileStore;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.*;
-import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * @author reghao
@@ -17,36 +13,50 @@ import java.util.concurrent.atomic.AtomicInteger;
  */
 public class LocalStores {
     private static final Map<String, LocalStore> storeMap = new HashMap<>();
-    private static final Map<String, Map<String, StoreDir>> subDirs = new HashMap<>();
+    private static final Map<String, Map<String, StoreDir>> storeDirMap = new HashMap<>();
 
-    public static void init(OssProperties ossProperties) throws IOException {
-        for (String baseDir : ossProperties.getMountedDirs()) {
-            FileStore fileStore = Files.getFileStore(Path.of(baseDir));
-            String fs = fileStore.name();
-            String mountedOn = fileStore.toString().replace(fs, "").replace(" ()", "");
+    public static void init(List<String> diskDirs) throws IOException {
+        Set<String> set = new HashSet<>();
+        for (String diskDir : diskDirs) {
+            FileStore fileStore = Files.getFileStore(Path.of(diskDir));
+            String logicalDisk = fileStore.name();
+            if (set.add(logicalDisk)) {
+                long total = fileStore.getTotalSpace();
+                long available = fileStore.getUsableSpace();
+                LocalStore localStore = new LocalStore(diskDir, total, available, 0.9);
+                storeMap.put(diskDir, localStore);
+                storeDirMap.computeIfAbsent(diskDir, k -> new HashMap<>());
+                createSubDirs(diskDir);
+            }
+        }
+    }
 
-            LocalStore localStore = new LocalStore(baseDir, fileStore, 0.9);
-            storeMap.put(mountedOn, localStore);
-            subDirs.computeIfAbsent(mountedOn, k -> new HashMap<>());
-            createSubDirs(mountedOn, baseDir);
+    public static void initStoreDirs(String diskDir, Map<String, Integer> map) {
+        Map<String, StoreDir> map1 = storeDirMap.get(diskDir);
+        if (map1 == null) {
+            return;
         }
+
+        map.forEach((path, total) -> {
+            StoreDir storeDir = map1.get(path);
+            if (storeDir != null) {
+                storeDir.setTotal(total);
+            }
+        });
     }
 
-    private static void createSubDirs(String blockId, String baseDir) throws IOException {
+    private static void createSubDirs(String diskDir) throws IOException {
         int total = 128;
         for (int i = 0; i < total; i++) {
             for (int j = 0; j < total; j++) {
-                String absoluteDirPath = String.format("%s%s/%s/", baseDir, i, j);
-                File dir = new File(absoluteDirPath);
-                int filesCount = 0;
-                if (!dir.exists()) {
-                    FileUtils.forceMkdir(dir);
-                } else {
-                    // 统计每个目录下文件的数量
-                    filesCount = Objects.requireNonNull(dir.list()).length;
+                String baseDir = String.format("%s%s/%s/", diskDir, i, j);
+                File dir = new File(baseDir);
+                if (!dir.exists() && !dir.mkdirs()) {
+                    String msg = String.format("create %s failed", dir);
+                    throw new IOException(msg);
                 }
-                StoreDir storeDir = new StoreDir(blockId, new AtomicInteger(filesCount), absoluteDirPath);
-                subDirs.get(blockId).put(absoluteDirPath, storeDir);
+
+                storeDirMap.get(diskDir).put(baseDir, new StoreDir(baseDir));
             }
         }
     }
@@ -55,25 +65,25 @@ public class LocalStores {
     public static LocalStore getMaxStore(long size) {
         Map<String, Long> map = new HashMap<>();
         for (Map.Entry<String, LocalStore> entry : storeMap.entrySet()) {
-            String blockId = entry.getKey();
+            String diskDir = entry.getKey();
             LocalStore localStore = entry.getValue();
-            long availSize = localStore.getAvail() - size;
-            map.put(blockId, availSize);
+            long currentAvailable = localStore.getAvailable() - size;
+            map.put(diskDir, currentAvailable);
         }
 
-        List<String> baseDirs = new ArrayList<>();
-        // mountedDirs 中的元素升序排列
+        List<String> diskDirs = new ArrayList<>();
+        // diskDirs 中的元素升序排列
         map.entrySet().stream()
                 .sorted(Map.Entry.comparingByValue())
-                .forEachOrdered(b -> baseDirs.add(b.getKey()));
+                .forEachOrdered(b -> diskDirs.add(b.getKey()));
 
-        String baseDir = baseDirs.get(baseDirs.size()-1);
-        LocalStore localStore = storeMap.get(baseDir);
-        localStore.setAvail(map.get(baseDir));
+        String maxDisk = diskDirs.get(diskDirs.size()-1);
+        LocalStore localStore = storeMap.get(maxDisk);
+        localStore.setCurrentAvailable(map.get(maxDisk));
         return localStore;
     }
     
-    public static List<StoreDir> getSubDirs(String blockId) {
-        return new ArrayList<>(subDirs.get(blockId).values());
+    public static List<StoreDir> getSubDirs(String diskDir) {
+        return new ArrayList<>(storeDirMap.get(diskDir).values());
     }
 }

oss-store/src/main/java/cn/reghao/oss/store/util/store/StoreDir.java

@@ -1,36 +1,37 @@
 package cn.reghao.oss.store.util.store;
 
-import lombok.AllArgsConstructor;
-
 import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * @author reghao
  * @date 2022-05-23 23:31:21
  */
-@AllArgsConstructor
 public class StoreDir {
-    private String blockId;
-    private volatile AtomicInteger filesCount;
-    private String absoluteDirPath;
+    private final String baseDir;
+    private final AtomicInteger total;
+
+    public StoreDir(String baseDir) {
+        this.baseDir = baseDir;
+        this.total = new AtomicInteger(0);
+    }
 
-    public String getBlockId() {
-        return blockId;
+    public String getBaseDir() {
+        return baseDir;
     }
 
-    public int getCount() {
-        return filesCount.get();
+    public void setTotal(int total) {
+        this.total.setPlain(total);
     }
 
-    public void incrCount() {
-        filesCount.incrementAndGet();
+    public int getTotal() {
+        return total.get();
     }
 
-    public void decrCount() {
-        filesCount.decrementAndGet();
+    public void incr() {
+        this.total.incrementAndGet();
     }
 
-    public String getAbsoluteDirPath() {
-        return absoluteDirPath;
+    public void decr() {
+        this.total.decrementAndGet();
     }
 }