update dfs-store

dfs-store/src/main/java/cn/reghao/dfs/store/util/media/FFmpegWrapper.java

@@ -38,13 +38,34 @@ public class FFmpegWrapper {
                     String codecName = jsonObject1.get("codec_name").getAsString();
                     String codecTagString = jsonObject1.get("codec_tag_string").getAsString();
                     double bitRate = jsonObject1.get("bit_rate").getAsDouble();
-                    double duration = jsonObject1.get("duration").getAsDouble();
+                    JsonElement durationElement = jsonObject1.get("duration");
+                    double duration;
+                    if (durationElement == null) {
+                        duration = 0;
+                    } else {
+                        duration = durationElement.getAsDouble();
+                    }
                     audioProps = new AudioProps(codecName, codecTagString, bitRate, duration);
                 } else if (codecType.equals("video")) {
                     String codecName = jsonObject1.get("codec_name").getAsString();
                     String codecTagString = jsonObject1.get("codec_tag_string").getAsString();
-                    double bitRate = jsonObject1.get("bit_rate").getAsDouble();
-                    double duration = jsonObject1.get("duration").getAsDouble();
+
+                    double bitRate;
+                    JsonElement biteRateElement = jsonObject1.get("bit_rate");
+                    if (biteRateElement == null) {
+                        bitRate = 0;
+                    } else {
+                        bitRate = biteRateElement.getAsDouble();
+                    }
+
+                    double duration;
+                    JsonElement durationElement = jsonObject1.get("duration");
+                    if (durationElement == null) {
+                        duration = 0;
+                    } else {
+                        duration = durationElement.getAsDouble();
+                    }
+
                     double codedWidth = jsonObject1.get("coded_width").getAsDouble();
                     double codedHeight = jsonObject1.get("coded_height").getAsDouble();
                     videoProps = new VideoProps(codecName, codecTagString, bitRate, duration, codedWidth, codedHeight);
@@ -57,14 +78,16 @@ public class FFmpegWrapper {
             double bitRate = format.get("bit_rate").getAsDouble();
 
             MediaProps mediaProps = new MediaProps(audioProps, videoProps);
-            JsonObject tags = format.get("tags").getAsJsonObject();
-            JsonElement jsonElement = tags.get("creation_time");
-            if (jsonElement != null) {
-                String creationTime = jsonElement.getAsString();
-                LocalDateTime localDateTime = DateTimeConverter.localDateTime(creationTime);
-                mediaProps.setCreateTime(localDateTime);
+            JsonElement tagsElement = format.get("tags");
+            if (tagsElement != null) {
+                JsonObject tags = tagsElement.getAsJsonObject();
+                JsonElement jsonElement = tags.get("creation_time");
+                if (jsonElement != null) {
+                    String creationTime = jsonElement.getAsString();
+                    LocalDateTime localDateTime = DateTimeConverter.localDateTime(creationTime);
+                    mediaProps.setCreateTime(localDateTime);
+                }
             }
-
             return mediaProps;
         }
         return null;

dfs-store/src/main/resources/mapper/DataBlockMapper.xml

@@ -31,6 +31,12 @@
                 </foreach>
             </trim>
         </trim>
+        <where>
+            object_id in
+            <foreach collection="list" item="item" separator="," open="(" close=")">
+                #{item.objectId}
+            </foreach>
+        </where>
     </update>
 
     <select id="findAll" resultType="cn.reghao.dfs.store.model.po.DataBlock">

dfs-store/src/main/resources/mapper/FileMetaMapper.xml

@@ -31,6 +31,12 @@
                 </foreach>
             </trim>
         </trim>
+        <where>
+            object_id in
+            <foreach collection="list" item="item" separator="," open="(" close=")">
+                #{item.objectId}
+            </foreach>
+        </where>
     </update>
 
     <select id="count" resultType="java.lang.Integer">

dfs-store/src/test/java/FileMetaTest.java

@@ -63,7 +63,7 @@ public class FileMetaTest {
 
     @Test
     public void test2() {
-        int pageSize = 1000;
+        int pageSize = 10000;
         int pageNumber = 1;
         Page page = new Page(pageNumber, pageSize);
         List<DataBlock> dataBlocks = dataBlockMapper.findDataBlockByPage(page);
@@ -113,15 +113,7 @@ public class FileMetaTest {
             @Override
             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                 String absolutePath = file.toString();
-                try {
-                    FileInputStream fis = new FileInputStream(absolutePath);
-                    String sha256sum = DigestUtil.sha256sum(fis);
-                    FileMeta fileMeta = fileMetaMapper.findBySha256sum(sha256sum);
-                    if (fileMeta == null) {
-                        log.error("{} 不存在", absolutePath);
-                    }
-                } catch (Exception ignore) {
-                }
+                process(absolutePath);
                 return FileVisitResult.CONTINUE;
             }
 
@@ -137,6 +129,17 @@ public class FileMetaTest {
         });
     }
 
+    private void process(String absolutePath) {
+        try {
+            String sha256sum = DigestUtil.sha256sum(absolutePath);
+            FileMeta fileMeta = fileMetaMapper.findBySha256sum(sha256sum);
+            if (fileMeta == null) {
+                log.error("{} 不存在", absolutePath);
+            }
+        } catch (Exception ignore) {
+        }
+    }
+
     @Test
     public void test22() throws IOException {
         String baseDir = "/home/reghao/mnt/zdata/porn/0.已完成/";

dfs-store/src/test/java/FileTest.java

@@ -0,0 +1,108 @@
+import cn.reghao.dfs.store.util.media.FFmpegWrapper;
+import cn.reghao.dfs.store.util.media.MediaQuality;
+import cn.reghao.dfs.store.util.media.MediaResolution;
+import cn.reghao.dfs.store.util.media.po.AudioProps;
+import cn.reghao.dfs.store.util.media.po.MediaProps;
+import cn.reghao.dfs.store.util.media.po.VideoProps;
+import cn.reghao.jutil.jdk.security.DigestUtil;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.FileVisitor;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author reghao
+ * @date 2023-03-22 16:19:12
+ */
+@Slf4j
+public class FileTest {
+    public void walkDir(Path path) throws IOException {
+        Files.walkFileTree(path, new FileVisitor<>() {
+            @Override
+            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+                return FileVisitResult.CONTINUE;
+            }
+
+            @Override
+            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                String absolutePath = file.toString();
+                //process(absolutePath);
+                unique(absolutePath);
+                return FileVisitResult.CONTINUE;
+            }
+
+            @Override
+            public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
+                return FileVisitResult.CONTINUE;
+            }
+
+            @Override
+            public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+                return FileVisitResult.CONTINUE;
+            }
+        });
+    }
+
+    private void process(String absolutePath) {
+        MediaProps mediaProps = FFmpegWrapper.getMediaProps(absolutePath);
+        if (mediaProps == null) {
+            log.error("{} 没有媒体信息", absolutePath);
+            return;
+        }
+
+        AudioProps audioProps = mediaProps.getAudioProps();
+        if (audioProps == null) {
+            log.error("{} 没有音频信息", absolutePath);
+            return;
+        }
+
+        VideoProps videoProps = mediaProps.getVideoProps();
+        if (videoProps == null) {
+            log.error("{} 没有视频信息", absolutePath);
+            return;
+        }
+
+        String audioCodec = audioProps.getCodecName();
+        String videoCodec = videoProps.getCodecName();
+        int width = (int) videoProps.getCodedWidth();
+        int height = (int) videoProps.getCodedHeight();
+        if ("aac".equals(audioCodec) && "h264".equals(videoCodec)) {
+            MediaResolution mediaResolution = MediaQuality.getQuality(width, height);
+        } else {
+            log.error("{} 不是 aac&h264", absolutePath);
+        }
+    }
+
+    Map<String, String> map = new HashMap<>();
+    List<String> list = new ArrayList<>();
+    private void unique(String absolutePath) {
+        try {
+            String sha256sum = DigestUtil.sha256sum(absolutePath);
+            String filePath = map.get(sha256sum);
+            if (filePath == null) {
+                map.put(sha256sum, filePath);
+            } else {
+                log.info("{} 已存在", filePath);
+                list.add(filePath);
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    @Test
+    public void test22() throws IOException {
+        String baseDir = "/home/reghao/mnt/porn/1.待发布/13996.反差婊系列/p/";
+        Path path = Path.of(baseDir);
+        walkDir(path);
+    }
+}