update

pom.xml

@@ -53,11 +53,6 @@
             <artifactId>dfs-api</artifactId>
             <version>1.0.0-SNAPSHOT</version>
         </dependency>
-        <dependency>
-            <groupId>cn.reghao.dfs</groupId>
-            <artifactId>dfs-client</artifactId>
-            <version>1.0.0-SNAPSHOT</version>
-        </dependency>
 
         <dependency>
             <groupId>org.projectlombok</groupId>

src/main/java/cn/reghao/dfs/store/controller/MediaUploadController.java

@@ -55,6 +55,8 @@ public class MediaUploadController {
 
         UploadingFile uploadingFile = new UploadingFile(file);
         UploadedFile uploadedFile = fileUploadService.put(uploadingFile);
+        ImageFileRet imageFileRet = imageFileService.process(uploadedFile);
+
         UploadFileRet uploadFileRet = new UploadFileRet(uploadedFile.getUploadId(), uploadedFile.getPathUrl().getUrl());
         return WebBody.success(uploadFileRet);
     }
@@ -69,10 +71,7 @@ public class MediaUploadController {
 
         UploadingFile uploadingFile = new UploadingFile(file);
         UploadedFile uploadedFile = fileUploadService.put(uploadingFile);
-        String fileId = uploadedFile.getFileId();
-        PathUrl pathUrl = uploadedFile.getPathUrl();
-
-        ImageFileRet imageFileRet = imageFileService.process(fileId, pathUrl);
+        ImageFileRet imageFileRet = imageFileService.process(uploadedFile);
         return WebBody.success(imageFileRet);
     }
 

src/main/java/cn/reghao/dfs/store/db/mapper/ImageUrlMapper.java

@@ -15,6 +15,7 @@ import java.util.List;
 public interface ImageUrlMapper extends BaseMapper<ImageUrl> {
     void updateSetUrl(@Param("fileId") String fileId, @Param("url") String url);
 
+
     @Deprecated
     List<String> getImageUrls();
     @Deprecated

src/main/java/cn/reghao/dfs/store/db/mapper/VideoFileMapper.java

@@ -11,5 +11,7 @@ import org.apache.ibatis.annotations.Param;
  */
 @Mapper
 public interface VideoFileMapper extends BaseMapper<VideoFile> {
-    void updateSetCover(@Param("videoFileId") String videoFileId, @Param("coverUrl") String coverUrl);
+    void updateSetCover(@Param("videoFileId") String videoFileId,
+                        @Param("coverUrl") String coverUrl,
+                        @Param("coverUrlOriginal") String coverUrlOriginal);
 }

src/main/java/cn/reghao/dfs/store/model/dto/UploadFilePart.java

@@ -6,6 +6,7 @@ import org.springframework.web.multipart.MultipartFile;
 
 import javax.validation.constraints.NotBlank;
 import javax.validation.constraints.NotNull;
+import java.io.InputStream;
 import java.io.Serializable;
 
 /**

src/main/java/cn/reghao/dfs/store/model/po/ImageFile.java

@@ -1,7 +1,6 @@
 package cn.reghao.dfs.store.model.po;
 
 import cn.reghao.jutil.jdk.db.BaseObject;
-import cn.reghao.dfs.client.model.SpiderImageFile;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.EqualsAndHashCode;

src/main/java/cn/reghao/dfs/store/model/po/VideoFile.java

@@ -1,7 +1,6 @@
 package cn.reghao.dfs.store.model.po;
 
 import cn.reghao.jutil.jdk.db.BaseObject;
-import cn.reghao.dfs.client.model.SpiderVideoFile;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
@@ -21,6 +20,7 @@ public class VideoFile extends BaseObject<Integer> {
     // 单位秒
     private Integer duration;
     private String coverUrl;
+    private String coverUrlOriginal;
 
     public VideoFile(String videoFileId) {
         this.videoFileId = videoFileId;

src/main/java/cn/reghao/dfs/store/model/vo/UploadPrepareRet.java

@@ -1,12 +1,14 @@
 package cn.reghao.dfs.store.model.vo;
 
 import lombok.AllArgsConstructor;
+import lombok.Getter;
 
 /**
  * @author reghao
  * @date 2022-04-21 09:27:55
  */
 @AllArgsConstructor
+@Getter
 public class UploadPrepareRet {
     private String uploadId;
     private long splitSize;

src/main/java/cn/reghao/dfs/store/model/vo/VideoFileRet.java

@@ -13,6 +13,7 @@ public class VideoFileRet {
     private Boolean horizontal;
     private Integer duration;
     private String coverUrl;
+    private String coverUrlOriginal;
 
     public VideoFileRet(String uploadId) {
         this.uploadId = uploadId;
@@ -26,5 +27,6 @@ public class VideoFileRet {
         this.duration = videoFile.getDuration();
         this.horizontal = videoFile.getHorizontal();
         this.coverUrl = videoFile.getCoverUrl();
+        this.coverUrlOriginal = videoFile.getCoverUrlOriginal();
     }
 }

src/main/java/cn/reghao/dfs/store/service/FileStoreService.java

@@ -26,19 +26,6 @@ public class FileStoreService {
         raf.close();
     }
 
-    public File createFile(String absolutePath, long len) throws IOException {
-        File file = new File(absolutePath);
-        if (!file.exists()) {
-            FileUtils.forceMkdirParent(file);
-            // 创建一个稀疏文件
-            Files.newByteChannel(Paths.get(absolutePath), EnumSet.of(CREATE_NEW, WRITE, SPARSE));
-            RandomAccessFile raf = new RandomAccessFile(file, "rw");
-            raf.setLength(len);
-            raf.close();
-        }
-        return file;
-    }
-
     public void writeToFile(InputStream in, String absolutePath, long pos) throws IOException {
         RandomAccessFile raf = new RandomAccessFile(absolutePath, "rw");
         raf.seek(pos);
@@ -51,40 +38,6 @@ public class FileStoreService {
         in.close();
     }
 
-    public void writeToFile(InputStream in, File file, long pos) throws IOException {
-        RandomAccessFile raf = new RandomAccessFile(file, "rw");
-        raf.seek(pos);
-        byte[] buf = new byte[1024*1024];
-        int len;
-        while ((len = in.read(buf)) != -1) {
-            raf.write(buf, 0, len);
-        }
-        raf.close();
-        in.close();
-    }
-
-    public void saveFile(String absolutePath, InputStream in) throws IOException {
-        File file = new File(absolutePath);
-        if (file.exists()) {
-            return;
-        }
-
-        File parentDir = file.getParentFile();
-        if (!parentDir.exists()) {
-            FileUtils.forceMkdir(parentDir);
-        }
-
-        FileOutputStream fos = new FileOutputStream(file);
-        // 1MiB
-        int len = 1024*1024;
-        byte[] buf = new byte[len];
-        int readLen;
-        while ((readLen = in.read(buf, 0, len)) != -1) {
-            fos.write(buf, 0, readLen);
-        }
-        fos.close();
-    }
-
     public void saveFile(String absolutePath, byte[] bytes) throws IOException {
         File file = new File(absolutePath);
         if (file.exists()) {

src/main/java/cn/reghao/dfs/store/service/media/ImageFileService.java

@@ -13,6 +13,7 @@ import cn.reghao.dfs.store.util.media.ImageOps;
 import cn.reghao.jutil.tool.id.IdGenerator;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
 
 import javax.imageio.ImageIO;
 import java.awt.image.BufferedImage;
@@ -39,9 +40,11 @@ public class ImageFileService {
         this.idGenerator = new IdGenerator("image-file-id");
     }
 
-    public ImageFileRet process(String fileId, PathUrl pathUrl) throws Exception {
+    public ImageFileRet process(UploadedFile uploadedFile) throws Exception {
+        String fileId = uploadedFile.getFileId();
+        PathUrl pathUrl = uploadedFile.getPathUrl();
         String filePath = pathUrl.getAbsolutePath();
-        String url = pathUrl.getUrl();
+        String originalUrl = pathUrl.getUrl();
 
         File originalFile = new File(filePath);
         ImageOps.Size imgSize = ImageOps.info(originalFile);
@@ -52,12 +55,32 @@ public class ImageFileService {
 
         imageFileMapper.save(imageFile);
         imageUrlMapper.save(imageUrl);
-        return genThumbnail(imageFileId, filePath, url);
+
+        long length = originalFile.length();
+        if (length < 1024*1024) {
+            return new ImageFileRet(uploadedFile.getUploadId(), imageFileId, pathUrl.getUrl(), originalUrl);
+        } else {
+            return genThumbnail(imageFileId, originalFile, originalUrl);
+        }
     }
 
-    private ImageFileRet genThumbnail(String imageFileId, String originalPath, String originalUrl) throws Exception {
-        BufferedImage bufferedImage = ImageIO.read(new FileInputStream(originalPath));
-        BufferedImage bufferedImage1 = ImageOps.resize(bufferedImage, 4);
+    private ImageFileRet genThumbnail(String imageFileId, File originalFile, String originalUrl) throws Exception {
+        long length = originalFile.length();
+        int ratio;
+        if (length > 1024*1024 && length < 1024*1024*2) {
+            ratio = 2;
+        } else if (length > 1024*1024*2 && length < 1024*1024*4) {
+            ratio = 4;
+        } else if (length > 1024*1024*4 && length < 1024*1024*8) {
+            ratio = 8;
+        } else if (length > 1024*1024*8 && length < 1024*1024*16) {
+            ratio = 16;
+        } else {
+            ratio = 32;
+        }
+
+        BufferedImage bufferedImage = ImageIO.read(new FileInputStream(originalFile));
+        BufferedImage bufferedImage1 = ImageOps.resize(bufferedImage, ratio);
 
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         ImageIO.write(bufferedImage1, "jpg", baos);
@@ -81,6 +104,7 @@ public class ImageFileService {
         return new ImageFileRet(uploadId, imageFileId, pathUrl.getUrl(), originalUrl);
     }
 
+    @Transactional(rollbackFor = Exception.class)
     public void saveImage(ImageFile imageFile, List<ImageUrl> imageUrls) {
         if (!imageUrls.isEmpty()) {
             imageFileMapper.save(imageFile);

src/main/java/cn/reghao/dfs/store/service/media/VideoFileService.java

@@ -56,15 +56,12 @@ public class VideoFileService {
      */
     @Transactional(rollbackFor = Exception.class)
     public VideoFile process(String fileId, PathUrl pathUrl) throws Exception {
-        String path = pathUrl.getAbsolutePath();
-        String url = pathUrl.getUrl();
-
         VideoFile videoFile = new VideoFile(idGenerator.getUuid());
-        File videoLocalFile = new File(path);
-        //log.info("process video {} with FFmpeg...", videoFile.getFileId());
+        File videoLocalFile = new File(pathUrl.getAbsolutePath());
+        log.info("process video {} with FFmpeg...", videoFile.getVideoFileId());
         setVideoProps(videoFile, fileId, videoLocalFile);
         setVideoCover(videoFile, videoLocalFile);
-        //log.info("video {} processed...", videoFile.getFileId());
+        log.info("video {} processed...", videoFile.getVideoFileId());
 
         videoFileMapper.save(videoFile);
         return videoFile;
@@ -124,15 +121,12 @@ public class VideoFileService {
         ImageUrl imageUrl = new ImageUrl(imageFileId, fileId, size.getWidth(), size.getHeight());
         imageFileService.saveImage(imageFile, List.of(imageUrl));
 
-        videoFile.setCoverUrl(uploadedFile.getPathUrl().getUrl());
+        String coverUrl = uploadedFile.getPathUrl().getUrl();
+        videoFile.setCoverUrl(coverUrl);
+        videoFile.setCoverUrlOriginal(coverUrl);
     }
 
     @Transactional(rollbackFor = Exception.class)
-    public String setVideoCover(String videoFileId, UploadingFile uploadingFile) throws Exception {
-        UploadedFile uploadedFile = fileUploadService.put(uploadingFile);
-        String url = uploadedFile.getPathUrl().getUrl();
-        /*videoFile.setCoverUrl(url);
-        videoFileMapper.updateSetCover(videoFileId, url);*/
-        return url;
+    public void setVideoCover(String videoFileId, UploadingFile uploadingFile) {
     }
 }

src/main/java/cn/reghao/dfs/store/util/media/ImageOps.java

@@ -70,9 +70,9 @@ public class ImageOps {
      * @return
      * @date 2021-08-18 下午1:45
      */
-    public static BufferedImage resize(BufferedImage srcImage, int size) {
-        int width = srcImage.getWidth()/size;
-        int height = srcImage.getHeight()/size;
+    public static BufferedImage resize(BufferedImage srcImage, int ratio) {
+        int width = srcImage.getWidth()/ratio;
+        int height = srcImage.getHeight()/ratio;
 
         BufferedImage newImage = new BufferedImage(width, height, srcImage.getType());
         Graphics g = newImage.getGraphics();

src/main/resources/mapper/ImageUrlMapper.xml

@@ -18,7 +18,7 @@
     </insert>
 
     <update id="updateSetUrl">
-        update image_url set url=#{url} where file_id=#{fileId}
+        update mblog_image set origin_url=#{url} where image_file_id=#{fileId}
     </update>
 
     <select id="findAll" resultType="cn.reghao.dfs.store.model.po.ImageUrl">

src/main/resources/mapper/VideoFileMapper.xml

@@ -4,13 +4,13 @@
 <mapper namespace="cn.reghao.dfs.store.db.mapper.VideoFileMapper">
     <insert id="save" useGeneratedKeys="true" keyProperty="id">
         insert into video_file
-        (`id`,`deleted`,`create_time`,`update_time`,`video_file_id`,`horizontal`,`duration`,`cover_url`)
+        (`id`,`deleted`,`create_time`,`update_time`,`video_file_id`,`horizontal`,`duration`,`cover_url`,`cover_url_original`)
         values 
-        (#{id},#{deleted},#{createTime},#{updateTime},#{videoFileId},#{horizontal},#{duration},#{coverUrl})
+        (#{id},#{deleted},#{createTime},#{updateTime},#{videoFileId},#{horizontal},#{duration},#{coverUrl},#{coverUrlOriginal})
     </insert>
 
     <update id="updateSetCover">
-        update video_file set update_time=now(), cover_url=#{coverUrl}
+        update video_file set update_time=now(),cover_url=#{coverUrl},cover_url_original=#{coverUrlOriginal}
         where file_id=#{fileId}
     </update>
 </mapper>

src/test/java/ConsistentCheckTest.java

@@ -25,6 +25,7 @@ import java.net.URI;
 import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
+import java.security.NoSuchAlgorithmException;
 import java.time.Duration;
 import java.util.*;
 
@@ -71,9 +72,9 @@ public class ConsistentCheckTest {
 
                 ByteArrayInputStream inputStream = new ByteArrayInputStream(imageBytes);
                 UploadingFile uploadingFile = new UploadingFile(filename, size, contentType, inputStream);
-                UploadedFile fileRet = fileUploadService.put(uploadingFile);
+                UploadedFile uploadedFile = fileUploadService.put(uploadingFile);
 
-                ImageFileRet imageFileRet = imageFileService.process(fileRet.getFileId(), fileRet.getPathUrl());
+                ImageFileRet imageFileRet = imageFileService.process(uploadedFile);
                 String imageFileId = imageFileRet.getImageFileId();
                 String url1 = imageFileRet.getThumbnailUrl();
                 imageUrlMapper.updateImageUrl(url, imageFileId, url1);
@@ -167,4 +168,19 @@ public class ConsistentCheckTest {
             videoUrlMapper.updateSetUrl(fileId, url);
         });
     }
+
+    @Test
+    public void uploadSmallImageTest() throws Exception {
+        String filePath = "/home/reghao/Downloads/pic/banner/bg.jpg";
+        File file = new File(filePath);
+
+        String filename = file.getName();
+        long size = file.length();
+        String contentType = "image/jpeg";
+        InputStream inputStream = new FileInputStream(file);
+
+        UploadingFile uploadingFile = new UploadingFile(filename, size, contentType, inputStream);
+        UploadedFile uploadedFile = fileUploadService.put(uploadingFile);
+        System.out.println(uploadedFile.getPathUrl().getUrl());
+    }
 }

src/test/java/FileTest.java

@@ -1,6 +1,9 @@
+import cn.reghao.dfs.store.util.media.ImageOps;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.Test;
 
+import javax.imageio.ImageIO;
+import java.awt.image.BufferedImage;
 import java.io.*;
 
 /**
@@ -10,8 +13,8 @@ import java.io.*;
 @Slf4j
 public class FileTest {
     @Test
-    public void test() throws IOException {
-        String dirPath = "/opt/file/disk0/spider/bili/img/vidcover";
+    public void avatarSizeTest() throws IOException {
+        String dirPath = "/opt/file/disk0/spider/weibo/img/avatar/";
         File dir = new File(dirPath);
         for (File file : dir.listFiles()) {
             if (file.isDirectory()) {
@@ -20,7 +23,19 @@ public class FileTest {
 
             long len = file.length();
             if (len > 1024*1024) {
-                log.error("{} 大小超过 1MiB", file.getAbsolutePath());
+                log.error("{} 大小超过 1MiB, 实际大小为 {}", file.getAbsolutePath(), len/1024);
+
+                /*String userId = file.getName().split("\\.")[0];
+
+                BufferedImage bufferedImage = ImageIO.read(new FileInputStream(file));
+                BufferedImage bufferedImage1 = ImageOps.resize(bufferedImage, 2);
+
+                String dest = String.format("/opt/file/disk0/spider/bili/img/avatar1/%s.jpg", userId);
+                File destFile = new File(dest);
+                ImageIO.write(bufferedImage1, "jpg", destFile);
+
+                long len1 = destFile.length();
+                log.error("{} 大小超过 1MiB, 实际大小为 {}, 调整后的大小为 {}", file.getAbsolutePath(), len/1024, len1/1024);*/
             }
         }
     }