使用 redis 缓存 FileMeta 和 DataBlock 相关数据

dfs-store/src/main/java/cn/reghao/dfs/store/db/mapper/FileMetaMapper.java

@@ -2,7 +2,9 @@ package cn.reghao.dfs.store.db.mapper;
 
 import cn.reghao.dfs.api.dto.HeadObjectResult;
 import cn.reghao.dfs.store.model.po.FileMeta;
+import cn.reghao.dfs.store.model.vo.ObjectMeta;
 import cn.reghao.jutil.jdk.db.BaseMapper;
+import cn.reghao.jutil.jdk.db.Page;
 import org.apache.ibatis.annotations.Mapper;
 import org.apache.ibatis.annotations.Param;
 
@@ -24,4 +26,6 @@ public interface FileMetaMapper extends BaseMapper<FileMeta> {
     List<FileMeta> findAll2(@Param("bucket") String bucket, @Param("prefix") String prefix,
                             @Param("start") String start, @Param("max") Integer max);
     String findByObjectId(String objectId);
+    ObjectMeta findObjectMeta(String objectName);
+    List<ObjectMeta> findObjectMetaByPage(Page page);
 }

dfs-store/src/main/java/cn/reghao/dfs/store/model/vo/ObjectMeta.java

@@ -0,0 +1,22 @@
+package cn.reghao.dfs.store.model.vo;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import java.io.Serializable;
+
+/**
+ * @author reghao
+ * @date 2023-04-30 12:17:28
+ */
+@Setter
+@Getter
+public class ObjectMeta implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+    private long size;
+    private String contentType;
+    private String objectName;
+    private String objectId;
+    private String absolutePath;
+}

dfs-store/src/main/java/cn/reghao/dfs/store/service/ObjectBasicService.java

@@ -6,6 +6,7 @@ import cn.reghao.dfs.store.db.mapper.FileMetaMapper;
 import cn.reghao.dfs.store.model.po.ContentRange;
 import cn.reghao.dfs.store.model.po.DataBlock;
 import cn.reghao.dfs.store.model.po.FileMeta;
+import cn.reghao.dfs.store.model.vo.ObjectMeta;
 import cn.reghao.dfs.store.redis.ds.RedisStringObj;
 import cn.reghao.jutil.jdk.security.DigestUtil;
 import cn.reghao.jutil.tool.id.IdGenerator;
@@ -150,14 +151,17 @@ public class ObjectBasicService {
         HttpServletResponse response = ServletUtil.getResponse();
         Object object = redisStringObj.get(objectName);
         FileMeta fileMeta;
+        ObjectMeta objectMeta;
         if (object != null) {
             fileMeta = (FileMeta) object;
+            objectMeta = (ObjectMeta) object;
         } else {
             fileMeta = fileMetaMapper.findByObjectName(objectName);
+            objectMeta = fileMetaMapper.findObjectMeta(objectName);
         }
 
         // FileMeta fileMeta = fileMetaMapper.findByObjectName(objectName);
-        if (fileMeta == null) {
+        if (objectMeta == null) {
             response.setStatus(HttpServletResponse.SC_NOT_FOUND);
 
             OutputStream outputStream = response.getOutputStream();
@@ -166,11 +170,11 @@ public class ObjectBasicService {
             return;
         }
 
-        String objectId = fileMeta.getObjectId();
-        String contentType = fileMeta.getContentType();
+        String objectId = objectMeta.getObjectId();
+        String contentType = objectMeta.getContentType();
         long len = fileMeta.getSize();
         if (len < partLength) {
-            writeWholeContent(objectId, contentType, len);
+            writeWholeContent(objectId, contentType, len, objectMeta.getAbsolutePath());
             return;
         }
 
@@ -180,10 +184,10 @@ public class ObjectBasicService {
             String[] arr = rangeStr.replace("bytes=", "").split("-");
             long start = Long.parseLong(arr[0]);
             if (arr.length == 1) {
-                writePartialContent(objectId, contentType, start, len);
+                writePartialContent(objectId, contentType, start, len, objectMeta.getAbsolutePath());
             } else {
                 ContentRange contentRange = parseContentRange(range, len);
-                byte[] bytes = range1(objectId, contentRange.getStart(), contentRange.getEnd());
+                byte[] bytes = range1(objectId, contentRange.getStart(), contentRange.getEnd(), objectMeta.getAbsolutePath());
                 writePartialContent(bytes, contentType, contentRange, len);
             }
         } else {
@@ -237,7 +241,7 @@ public class ObjectBasicService {
         outputStream.close();
     }
 
-    private void writePartialContent(String objectId, String contentType, long start, long len) throws IOException {
+    private void writePartialContent(String objectId, String contentType, long start, long len, String filePath) throws IOException {
         HttpServletResponse response = ServletUtil.getResponse();
         response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
         response.setContentType(contentType);
@@ -251,12 +255,12 @@ public class ObjectBasicService {
         if (cachePath != null && Files.exists(Path.of(cachePath))) {
             raf = new RandomAccessFile(cachePath, "r");
         } else {
-            List<DataBlock> list = dataBlockMapper.findByObjectId(objectId);
+            /*List<DataBlock> list = dataBlockMapper.findByObjectId(objectId);
             DataBlock dataBlock = list.get(0);
             String blockId = dataBlock.getBlockId();
-            String absolutePath = dataBlock.getAbsolutePath();
-            localCache.putCache(objectId, absolutePath);
-            raf = new RandomAccessFile(absolutePath, "r");
+            String absolutePath = dataBlock.getAbsolutePath();*/
+            //localCache.putCache(objectId, filePath);
+            raf = new RandomAccessFile(filePath, "r");
         }
 
         /*List<DataBlock> list = dataBlockMapper.findByObjectId(objectId);
@@ -279,6 +283,13 @@ public class ObjectBasicService {
         raf.close();
     }
 
+    /**
+     * 下载内容
+     *
+     * @param
+     * @return
+     * @date 2023-04-30 1:12 AM
+     */
     private void writeDownloadContent(String objectName, String objectId, long start, long len) {
         HttpServletResponse response = ServletUtil.getResponse();
         response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
@@ -314,18 +325,25 @@ public class ObjectBasicService {
         }
     }
 
-    private void writeWholeContent(String objectId, String contentType, long len) throws IOException {
+    /**
+     * 返回内容
+     *
+     * @param
+     * @return
+     * @date 2023-04-30 1:12 AM
+     */
+    private void writeWholeContent(String objectId, String contentType, long len, String filePath) throws IOException {
         HttpServletResponse response = ServletUtil.getResponse();
         response.setStatus(HttpServletResponse.SC_OK);
         response.setContentType(contentType);
         //response.setHeader("Content-Length", ""+len);
 
         OutputStream outputStream = response.getOutputStream();
-        List<DataBlock> list = dataBlockMapper.findByObjectId(objectId);
+        /*List<DataBlock> list = dataBlockMapper.findByObjectId(objectId);
         DataBlock dataBlock = list.get(0);
         String blockId = dataBlock.getBlockId();
-        String absolutePath = dataBlock.getAbsolutePath();
-        RandomAccessFile raf = new RandomAccessFile(absolutePath, "r");
+        String absolutePath = dataBlock.getAbsolutePath();*/
+        RandomAccessFile raf = new RandomAccessFile(filePath, "r");
 
         // 1MiB
         int bufSize = 1024*1024;
@@ -339,9 +357,9 @@ public class ObjectBasicService {
         raf.close();
     }
 
-    private byte[] range1(String objectId, long start, long end) throws IOException {
-        String absolutePath = dataBlockMapper.findByObjectId(objectId).get(0).getAbsolutePath();
-        RandomAccessFile raf = new RandomAccessFile(absolutePath, "r");
+    private byte[] range1(String objectId, long start, long end, String filePath) throws IOException {
+        // String absolutePath = dataBlockMapper.findByObjectId(objectId).get(0).getAbsolutePath();
+        RandomAccessFile raf = new RandomAccessFile(filePath, "r");
         raf.seek(start);
         long bufSize = end - start;
         byte[] bytes = new byte[(int) bufSize];

dfs-store/src/main/java/cn/reghao/dfs/store/util/mysql/DataSourceConfig.java

@@ -0,0 +1,68 @@
+package cn.reghao.dfs.store.util.mysql;
+
+import org.apache.ibatis.plugin.Interceptor;
+import org.apache.ibatis.session.SqlSessionFactory;
+import org.mybatis.spring.SqlSessionFactoryBean;
+import org.mybatis.spring.SqlSessionTemplate;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
+import org.springframework.jdbc.datasource.DataSourceTransactionManager;
+import org.springframework.transaction.PlatformTransactionManager;
+
+import javax.sql.DataSource;
+
+/**
+ * MyBatis 初始化配置
+ * 若配置了多数据源，则不启用此配置
+ *
+ * @author reghao
+ * @date 2021-04-26 17:48:29
+ */
+@Configuration
+public class DataSourceConfig {
+    private final DataSource dataSource;
+
+    public DataSourceConfig(DataSource dataSource) {
+        this.dataSource = dataSource;
+    }
+
+    @Bean
+    public SqlSessionFactory sqlSessionFactory() throws Exception {
+        SqlSessionFactoryBean factoryBean = new SqlSessionFactoryBean();
+        factoryBean.setDataSource(dataSource);
+        factoryBean.setPlugins(new Interceptor[]{pageListInterceptor()});
+
+        org.apache.ibatis.session.Configuration configuration = new org.apache.ibatis.session.Configuration();
+        configuration.setMapUnderscoreToCamelCase(true);
+        configuration.setDefaultFetchSize(100);
+        configuration.setDefaultStatementTimeout(30);
+        factoryBean.setConfiguration(configuration);
+
+        String location = "classpath*:mapper/**/**.xml";
+        factoryBean.setMapperLocations(new PathMatchingResourcePatternResolver().getResources(location));
+        return factoryBean.getObject();
+    }
+
+    /**
+     * 配置 mybatis 的分页拦截器
+     *
+     * @param
+     * @return
+     * @date 2021-12-21 下午5:23
+     */
+    @Bean
+    public PageListInterceptor pageListInterceptor() {
+        return new PageListInterceptor();
+    }
+
+    @Bean(value = "sqlSession")
+    public SqlSessionTemplate sqlSessionTemplate(SqlSessionFactory sqlSessionFactory) {
+        return new SqlSessionTemplate(sqlSessionFactory);
+    }
+
+    @Bean(value = "transactionManager")
+    public PlatformTransactionManager annotationDrivenTransactionManager() {
+        return new DataSourceTransactionManager(dataSource);
+    }
+}

dfs-store/src/main/java/cn/reghao/dfs/store/util/mysql/PageListInterceptor.java

@@ -0,0 +1,83 @@
+package cn.reghao.dfs.store.util.mysql;
+
+import cn.reghao.jutil.jdk.db.Page;
+import org.apache.ibatis.binding.MapperMethod;
+import org.apache.ibatis.executor.parameter.ParameterHandler;
+import org.apache.ibatis.executor.statement.StatementHandler;
+import org.apache.ibatis.mapping.MappedStatement;
+import org.apache.ibatis.plugin.*;
+import org.apache.ibatis.reflection.MetaObject;
+import org.apache.ibatis.reflection.SystemMetaObject;
+
+import java.sql.Connection;
+import java.util.Properties;
+
+/**
+ * MyBatis 分页拦截器
+ *
+ * @author reghao
+ * @date 2021-04-26 17:40:32
+ */
+@Intercepts({@Signature(type=StatementHandler.class,method="prepare",args={Connection.class,Integer.class})})
+public class PageListInterceptor implements Interceptor {
+    private final String methodSuffix = "ByPage";
+    @Deprecated
+    private int page;
+    @Deprecated
+    private int size;
+    private String dbType;
+
+    @Override
+    public Object intercept(Invocation invocation) throws Throwable {
+        StatementHandler statementHandler = (StatementHandler)invocation.getTarget();
+        MetaObject metaObject = SystemMetaObject.forObject(statementHandler);
+        while(metaObject.hasGetter("h")){
+            Object object = metaObject.getValue("h");
+            metaObject = SystemMetaObject.forObject(object);
+        }
+
+        while(metaObject.hasGetter("target")){
+            Object object = metaObject.getValue("target");
+            metaObject = SystemMetaObject.forObject(object);
+        }
+
+        MappedStatement mappedStatement = (MappedStatement)metaObject.getValue("delegate.mappedStatement");
+        String mapId = mappedStatement.getId();
+        if (mapId.matches(String.format(".+%s$", methodSuffix))){
+            ParameterHandler parameterHandler = (ParameterHandler)metaObject.getValue("delegate.parameterHandler");
+            Object object = parameterHandler.getParameterObject();
+            Page page;
+            if (object instanceof Page) {
+                page = (Page) object;
+            } else if (object instanceof MapperMethod.ParamMap) {
+                MapperMethod.ParamMap paramMap = (MapperMethod.ParamMap) object;
+                Object param1 = paramMap.get("param1");
+                if (param1 instanceof Page) {
+                    page = (Page) param1;
+                } else {
+                    throw new Exception("byPage 方法的第一个参数不是 Page 类型");
+                }
+            } else {
+                throw new Exception("没有 Page 类型的参数");
+            }
+
+            String sql = (String) metaObject.getValue("delegate.boundSql.sql");
+            //sql += " limit "+(page-1)*size +","+size;
+            sql += String.format(" limit %s,%s", (page.getPage()-1)*page.getSize(), page.getSize());
+            metaObject.setValue("delegate.boundSql.sql", sql);
+        }
+        return invocation.proceed();
+    }
+
+    @Override
+    public Object plugin(Object target) {
+        return Plugin.wrap(target, this);
+    }
+
+    @Override
+    public void setProperties(Properties properties) {
+        String limit = properties.getProperty("limit","10");
+        this.page = Integer.parseInt(limit);
+        this.dbType = properties.getProperty("dbType", "mysql");
+    }
+}

dfs-store/src/main/resources/mapper/FileMetaMapper.xml

@@ -70,4 +70,17 @@
         select upload_id from file_user
         where file_id=#{objectId}
     </select>
+    <select id="findObjectMeta" resultType="cn.reghao.dfs.store.model.vo.ObjectMeta">
+        select file_meta.size,file_meta.content_type,file_meta.object_name,data_block.absolute_path
+        from file_meta
+        inner join data_block
+        on file_meta.object_id=data_block.object_id
+        where file_meta.object_name=#{objectName}
+    </select>
+    <select id="findObjectMetaByPage" resultType="cn.reghao.dfs.store.model.vo.ObjectMeta">
+        select file_meta.size,file_meta.content_type,file_meta.object_name,data_block.absolute_path
+        from file_meta
+        inner join data_block
+        on file_meta.object_id=data_block.object_id
+    </select>
 </mapper>

dfs-store/src/test/java/RedisTest.java

@@ -1,8 +1,13 @@
 import cn.reghao.dfs.store.DfsStoreApplication;
+import cn.reghao.dfs.store.db.mapper.DataBlockMapper;
 import cn.reghao.dfs.store.db.mapper.FileMetaMapper;
+import cn.reghao.dfs.store.model.po.DataBlock;
 import cn.reghao.dfs.store.model.po.FileMeta;
+import cn.reghao.dfs.store.model.vo.ObjectMeta;
 import cn.reghao.dfs.store.redis.ds.RedisString;
 import cn.reghao.dfs.store.redis.ds.RedisStringObj;
+import cn.reghao.jutil.jdk.db.Page;
+import lombok.extern.slf4j.Slf4j;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -16,6 +21,7 @@ import java.util.List;
  * @author reghao
  * @date 2023-03-22 16:19:12
  */
+@Slf4j
 @ActiveProfiles("dev")
 @SpringBootTest(classes = DfsStoreApplication.class)
 @RunWith(SpringRunner.class)
@@ -26,6 +32,8 @@ public class RedisTest {
     RedisStringObj redisString;
     @Autowired
     FileMetaMapper fileMetaMapper;
+    @Autowired
+    DataBlockMapper dataBlockMapper;
     @Test
     public void test() {
         int total = fileMetaMapper.count();
@@ -36,4 +44,31 @@ public class RedisTest {
             redisString.set(key1, fileMeta);
         });
     }
+
+    @Test
+    public void test1() {
+        int pageSize = 10000;
+        int pageNumber = 1;
+        Page page = new Page(pageNumber, pageSize);
+        List<ObjectMeta> list = fileMetaMapper.findObjectMetaByPage(page);
+        while (!list.isEmpty()) {
+            list.forEach(objectMeta -> {
+                /*List<DataBlock> list1 = dataBlockMapper.findByObjectId(objectMeta.getObjectId());
+                if (list1.isEmpty()) {
+                    return;
+                }*
+                String absolutePath = list1.get(0).getAbsolutePath();
+                objectMeta.setAbsolutePath(absolutePath);*/
+
+                String objectName = objectMeta.getObjectName();
+                String key1 = key + objectName;
+                redisString.set(key1, objectMeta);
+            });
+
+            pageNumber++;
+            page = new Page(pageNumber, pageSize);
+            list = fileMetaMapper.findObjectMetaByPage(page);
+            log.info("page -> {}", pageNumber);
+        }
+    }
 }