web 模块默认不扫描 web.admin, web.console, web.parser

web/src/main/java/cn/reghao/bnt/web/WebApplication.java

@@ -12,7 +12,9 @@ import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
 @EnableJpaRepositories
 @EntityScan({"cn.reghao.bnt.web"})
 @ComponentScan(basePackages = {"cn.reghao.bnt.web", "cn.reghao.bnt.common"}
-		//,excludeFilters = @ComponentScan.Filter(type = FilterType.REGEX, pattern = {"cn.reghao.bnt.web.admin.*"})
+		,excludeFilters = @ComponentScan.Filter(type = FilterType.REGEX,
+		pattern = {"cn.reghao.bnt.web.admin.*", "cn.reghao.bnt.web.console.*",
+				"cn.reghao.bnt.web.parser.*", "org.springframework.data.mongodb.*"})
 )
 //@ConditionalOnProperty()
 public class WebApplication {

web/src/main/java/cn/reghao/bnt/web/blog/BlogController.java

@@ -1,145 +1,8 @@
 package cn.reghao.bnt.web.blog;
 
-import cn.reghao.bnt.web.sys.service.FileService;
-import cn.reghao.jutil.web.ServletUtil;
-import org.springframework.data.domain.PageRequest;
-import org.springframework.data.domain.Pageable;
-import org.springframework.data.domain.Sort;
-import org.springframework.stereotype.Controller;
-import org.springframework.ui.ModelMap;
-import org.springframework.web.bind.ServletRequestUtils;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.ResponseBody;
-
-import javax.servlet.http.HttpServletRequest;
-import java.io.IOException;
-import java.net.URLDecoder;
-import java.nio.charset.StandardCharsets;
-
 /**
  * @author reghao
  * @date 2024-07-24 13:50:19
  */
-@Controller
 public class BlogController {
-    private final FileService fileService;
-
-    public BlogController(FileService fileService) {
-        this.fileService = fileService;
-    }
-
-    @GetMapping(value = {"/blog", "/blog/index"})
-    public String index(ModelMap model, HttpServletRequest request) {
-        // 调用 ContentsDirective
-        String order = ServletRequestUtils.getStringParameter(request, "order", "newest");
-        int pageNo = ServletRequestUtils.getIntParameter(request, "pageNo", 1);
-        //model.put("user", accountQuery.getUserCard());
-        model.put("order", order);
-        model.put("pageNo", pageNo);
-
-        setChannels(model);
-        return "/blog/index";
-    }
-
-    @GetMapping("/category")
-    public String category(ModelMap model) {
-        /*List<CategoryCount> list = categoryService.findCategoryCountByPage(CategoryType.Category.getValue(), 1, 100).getContent();
-        model.put("results", list);
-
-        setChannels(model);*/
-        return "/blog/index";
-    }
-
-    @GetMapping("/category/{name}")
-    public String category1(@PathVariable("name") String name, ModelMap model) {
-        /*int pageNo = ServletRequestUtils.getIntParameter(ServletUtil.getRequest(), "pageNo", 1);
-        Page<UserArticle> page = articleQuery.findByPage1(10, pageNo, name, 1);
-        model.put("results", page);
-        model.put("name", name);
-        model.put("user", accountQuery.getUserCard());
-
-        setChannels(model);*/
-        return "/blog/index";
-    }
-
-    @GetMapping("/tag")
-    public String tag(ModelMap model) {
-        /*List<CategoryCount> list = categoryService.findCategoryCountByPage(CategoryType.Tag.getValue(), 1, 100).getContent();
-        model.put("results", list);
-
-        setChannels(model);*/
-        return "/blog/index";
-    }
-
-    @GetMapping("/tag/{name}")
-    public String tag1(@PathVariable String name, ModelMap model) {
-        /*int pageNo = ServletRequestUtils.getIntParameter(ServletUtil.getRequest(), "pageNo", 1);
-        Page<UserArticle> page = articleQuery.findByPage1(10, pageNo, name, 2);
-        model.put("results", page);
-        model.put("name", name);
-        model.put("user", accountQuery.getUserCard());
-
-        setChannels(model);*/
-        return "/blog/index";
-    }
-
-    @GetMapping("/archive")
-    public String archive(ModelMap model) {
-        /*List<ArchiveArticle> results = articleQuery.getArchiveArticles();
-        model.put("user", accountQuery.getUserCard());
-        model.put("results", results);
-
-        setChannels(model);*/
-        return "/blog/index";
-    }
-
-    @GetMapping("/post/{articleId}")
-    public String article(@PathVariable("articleId") String articleId, ModelMap model) {
-        /*ArticleVO articleVO = articleQuery.getArticle(articleId);
-        if (articleVO == null) {
-            model.put("title", "Not Found");
-            model.put("content", "文章不存在");
-            setChannels(model);
-            return "/classic/zzz/notfound";
-        }
-
-        articleViewService.incr(articleId);
-        model.put("user", accountQuery.getUserCard());
-        model.put("view", articleVO);
-        setChannels(model);*/
-        return "/blog/index";
-    }
-
-    @GetMapping("/search")
-    public String search(String kw, ModelMap model) {
-        Pageable pageable = PageRequest.of(0, 10, Sort.by(Sort.Direction.ASC, "createTime"));
-        /*Page<UserArticle> page = articleSearch.search(kw, pageable);
-
-        model.put("results", page);
-        model.put("kw", kw);
-        model.put("user", accountQuery.getUserCard());
-
-        setChannels(model);*/
-        return "/blog/index";
-    }
-
-    @GetMapping("/file/**")
-    @ResponseBody
-    public void getFile() throws IOException {
-        HttpServletRequest servletRequest = ServletUtil.getRequest();
-        String uri = servletRequest.getRequestURI();
-        String uri1 = URLDecoder.decode(uri, StandardCharsets.UTF_8);
-        String objectName =  uri1.replaceFirst("/", "");
-        fileService.getFile(objectName);
-    }
-
-    private void setChannels(ModelMap model) {
-        /*List<Channel> channels = new ArrayList<>();
-        channels.add(new Channel("分类", "category"));
-        channels.add(new Channel("标签", "tag"));
-        channels.add(new Channel("归档", "archive"));
-        model.put("searchBox", true);
-        model.put("channels", channels);*/
-    }
 }

web/src/main/java/cn/reghao/bnt/web/config/spring/AppLifecycle.java → web/src/main/java/cn/reghao/bnt/web/config/AppLifecycle.java

@@ -1,4 +1,4 @@
-package cn.reghao.bnt.web.config.spring;
+package cn.reghao.bnt.web.config;
 
 import cn.reghao.bnt.web.account.service.AccountService;
 import cn.reghao.bnt.web.config.AppProperties;

web/src/main/java/cn/reghao/bnt/web/config/SwaggerConfig.java

@@ -27,7 +27,7 @@ public class SwaggerConfig {
         return new Docket(DocumentationType.SWAGGER_2)
                 .apiInfo(apiInfo())
                 .select()
-                .apis(RequestHandlerSelectors.basePackage("cn.reghao.bnt.manager"))
+                .apis(RequestHandlerSelectors.basePackage("cn.reghao.bnt.web.devops"))
                 //.apis(RequestHandlerSelectors.any())
                 .paths(PathSelectors.any())
                 .build();
@@ -35,7 +35,7 @@ public class SwaggerConfig {
 
     private ApiInfo apiInfo() {
         return new ApiInfoBuilder()
-                .title("Manager APIs")
+                .title("devops APIs")
                 .description("")
                 .termsOfServiceUrl("")
                 .version("1.0.0")

web/src/main/java/cn/reghao/bnt/web/config/web/ControllerErrorHandler.java → web/src/main/java/cn/reghao/bnt/web/config/exception/ControllerErrorHandler.java

@@ -1,4 +1,4 @@
-package cn.reghao.bnt.web.config.web;
+package cn.reghao.bnt.web.config.exception;
 
 import io.swagger.annotations.Api;
 import lombok.extern.slf4j.Slf4j;

web/src/main/java/cn/reghao/bnt/web/config/web/ControllerExceptionHandler.java → web/src/main/java/cn/reghao/bnt/web/config/exception/ControllerExceptionHandler.java

@@ -1,4 +1,4 @@
-package cn.reghao.bnt.web.config.web;
+package cn.reghao.bnt.web.config.exception;
 
 import cn.reghao.jutil.jdk.exception.ExceptionUtil;
 import cn.reghao.jutil.jdk.result.WebResult;

web/src/main/java/cn/reghao/bnt/web/config/spring/BeansConfig.java

@@ -32,17 +32,4 @@ public class BeansConfig {
     public ByteConverter byteConverter() {
         return new ByteConverter();
     }
-
-    @Bean
-    public FileCache fileCache(AppProperties appProperties) {
-        return new FileCache(appProperties.getCacheDir());
-    }
-
-    @Bean
-    public OssConsoleConfig ossConsoleConfig() {
-        String endpoint = System.getenv("CONSOLE_ENDPOINT");
-        String accessKeyId = System.getenv("CONSOLE_ACCESS_KEY_ID");
-        String accessKeySecret = System.getenv("CONSOLE_ACCESS_KEY_SECRET");
-        return new OssConsoleConfig(endpoint, accessKeyId, accessKeySecret);
-    }
 }

web/src/main/java/cn/reghao/bnt/web/console/config/ConsoleConfig.java

@@ -0,0 +1,20 @@
+package cn.reghao.bnt.web.console.config;
+
+import cn.reghao.oss.sdk.model.OssConsoleConfig;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * @author reghao
+ * @date 2024-07-29 16:47:14
+ */
+@Configuration
+public class ConsoleConfig {
+    @Bean
+    public OssConsoleConfig ossConsoleConfig() {
+        String endpoint = System.getenv("CONSOLE_ENDPOINT");
+        String accessKeyId = System.getenv("CONSOLE_ACCESS_KEY_ID");
+        String accessKeySecret = System.getenv("CONSOLE_ACCESS_KEY_SECRET");
+        return new OssConsoleConfig(endpoint, accessKeyId, accessKeySecret);
+    }
+}

web/src/main/java/cn/reghao/bnt/web/config/OssConsoleClientFactory.java → web/src/main/java/cn/reghao/bnt/web/console/config/OssConsoleClientFactory.java

@@ -1,4 +1,4 @@
-package cn.reghao.bnt.web.config;
+package cn.reghao.bnt.web.console.config;
 
 import cn.reghao.oss.sdk.OssConsoleClient;
 import cn.reghao.oss.sdk.model.OssConsoleConfig;

web/src/main/java/cn/reghao/bnt/web/config/MongoConfig.java → web/src/main/java/cn/reghao/bnt/web/parser/config/MongoConfig.java

@@ -1,4 +1,4 @@
-package cn.reghao.bnt.web.config;
+package cn.reghao.bnt.web.parser.config;
 
 import com.mongodb.MongoClientSettings;
 import com.mongodb.MongoCredential;

web/src/main/java/cn/reghao/bnt/web/parser/config/ParserConfig.java

@@ -0,0 +1,18 @@
+package cn.reghao.bnt.web.parser.config;
+
+import cn.reghao.bnt.web.config.AppProperties;
+import cn.reghao.bnt.web.parser.util.FileCache;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * @author reghao
+ * @date 2024-07-29 16:47:39
+ */
+@Configuration
+public class ParserConfig {
+    @Bean
+    public FileCache fileCache(AppProperties appProperties) {
+        return new FileCache(appProperties.getCacheDir());
+    }
+}

web/src/main/java/cn/reghao/bnt/web/parser/site/bilibili/BiliCommentDataParser.java

@@ -1,6 +1,6 @@
 package cn.reghao.bnt.web.parser.site.bilibili;
 
-import cn.reghao.bnt.web.config.OssConsoleClientFactory;
+import cn.reghao.bnt.web.console.config.OssConsoleClientFactory;
 import cn.reghao.jutil.jdk.http.WebRequest;
 import cn.reghao.jutil.jdk.serializer.JsonConverter;
 import cn.reghao.jutil.tool.http.DefaultWebRequest;

web/src/main/java/cn/reghao/bnt/web/parser/task/DataConsumer.java

@@ -1,6 +1,6 @@
 package cn.reghao.bnt.web.parser.task;
 
-import cn.reghao.bnt.web.config.OssConsoleClientFactory;
+import cn.reghao.bnt.web.console.config.OssConsoleClientFactory;
 import cn.reghao.jutil.jdk.http.WebRequest;
 import cn.reghao.jutil.jdk.http.WebResponse;
 import cn.reghao.jutil.jdk.result.Result;
@@ -25,7 +25,6 @@ import com.google.gson.JsonObject;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
 
-import javax.annotation.PostConstruct;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ThreadPoolExecutor;
 

web/src/main/java/cn/reghao/bnt/web/parser/task/cache/CacheContext.java

@@ -1,9 +1,8 @@
 package cn.reghao.bnt.web.parser.task.cache;
 
-import cn.reghao.bnt.web.config.OssConsoleClientFactory;
+import cn.reghao.bnt.web.console.config.OssConsoleClientFactory;
 import cn.reghao.jutil.jdk.thread.ThreadPoolWrapper;
 import cn.reghao.oss.sdk.OssConsoleClient;
-import cn.reghao.oss.sdk.model.OssConsoleConfig;
 import cn.reghao.oss.api.rest.UploadFileRet;
 import cn.reghao.bnt.web.parser.site.bilibili.model.vo.BiliDash;
 import cn.reghao.bnt.web.parser.task.TnbService;

web/src/main/java/cn/reghao/bnt/web/parser/task/feed/FeedService.java

@@ -1,6 +1,6 @@
 package cn.reghao.bnt.web.parser.task.feed;
 
-import cn.reghao.bnt.web.config.OssConsoleClientFactory;
+import cn.reghao.bnt.web.console.config.OssConsoleClientFactory;
 import cn.reghao.jutil.jdk.converter.DateTimeConverter;
 import cn.reghao.jutil.jdk.http.WebRequest;
 import cn.reghao.jutil.tool.http.DefaultWebRequest;

web/src/main/java/cn/reghao/bnt/web/parser/task/feed/FeedTask.java

@@ -1,6 +1,6 @@
 package cn.reghao.bnt.web.parser.task.feed;
 
-import cn.reghao.bnt.web.config.OssConsoleClientFactory;
+import cn.reghao.bnt.web.console.config.OssConsoleClientFactory;
 import cn.reghao.jutil.jdk.converter.DateTimeConverter;
 import cn.reghao.jutil.jdk.http.WebRequest;
 import cn.reghao.jutil.tool.http.DefaultWebRequest;

web/src/main/java/cn/reghao/bnt/web/sys/controller/FileController.java

@@ -12,6 +12,9 @@ import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
 
 import javax.servlet.http.HttpServletRequest;
+import java.io.IOException;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -21,7 +24,6 @@ import java.util.Map;
  * @date 2024-01-20 17:24:58
  */
 @Controller
-@RequestMapping("/admin/file")
 public class FileController {
     private final FileService fileService;
 
@@ -29,7 +31,17 @@ public class FileController {
         this.fileService = fileService;
     }
 
-    @GetMapping("/list")
+    @GetMapping("/file/**")
+    @ResponseBody
+    public void getFile() throws IOException {
+        HttpServletRequest servletRequest = ServletUtil.getRequest();
+        String uri = servletRequest.getRequestURI();
+        String uri1 = URLDecoder.decode(uri, StandardCharsets.UTF_8);
+        String objectName =  uri1.replaceFirst("/", "");
+        fileService.getFile(objectName);
+    }
+
+    @GetMapping("/admin/file/list")
     public String list(ModelMap model, HttpServletRequest request) {
         int pageNumber = Integer.parseInt(ServletUtil.getRequestParam("pageNo", "1"));
         Page<DiskFile> page = fileService.getDiskFiles(pageNumber);
@@ -42,13 +54,13 @@ public class FileController {
         return "/admin/file/imglist";
     }
 
-    @GetMapping("/upload")
+    @GetMapping("/admin/file/upload")
     public String upload(ModelMap model, HttpServletRequest request) {
         model.put("id", 1);
         return "/admin/file/fileupload";
     }
 
-    @PostMapping(value = "/upload/image", produces = MediaType.APPLICATION_JSON_VALUE)
+    @PostMapping(value = "/admin/file/upload/image", produces = MediaType.APPLICATION_JSON_VALUE)
     @ResponseBody
     public String uploadImage(MultipartFile file) throws Exception {
         String url = fileService.putFile(file);
@@ -58,7 +70,7 @@ public class FileController {
         return WebResult.success(map);
     }
 
-    @PostMapping(value = "/upload/images", produces = MediaType.APPLICATION_JSON_VALUE)
+    @PostMapping(value = "/admin/file/upload/images", produces = MediaType.APPLICATION_JSON_VALUE)
     @ResponseBody
     public String uploadImages(@RequestParam("files") List<MultipartFile> files) {
         files.forEach(file -> {

web/src/main/java/cn/reghao/bnt/web/ws/event/MachineEventService.java

@@ -11,6 +11,7 @@ import cn.reghao.bnt.web.devops.machine.model.po.MachineHost;
 import cn.reghao.bnt.web.devops.machine.model.po.MachineInfo;
 import cn.reghao.jutil.jdk.machine.data.detail.DiskDetail;
 import cn.reghao.jutil.jdk.machine.data.detail.MemoryDetail;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
@@ -20,6 +21,7 @@ import java.util.List;
  * @author reghao
  * @date 2021-10-15 16:39:48
  */
+@Slf4j
 @Service
 public class MachineEventService {
     private final MachineInfoRepository machineInfoRepository;
@@ -71,5 +73,11 @@ public class MachineEventService {
         MemoryDetail memoryDetail = evtAgentHeartbeat.getMemoryDetail();
         List<DiskDetail> diskDetails = evtAgentHeartbeat.getDiskDetails();
         MachineInfo machineInfo = machineInfoRepository.findByMachineId(machineId);
+        if (machineInfo != null) {
+            String machineIpv4 = machineInfo.getMachineIpv4();
+            log.info("heartbeat from {}", machineIpv4);
+        } else {
+            log.info("heartbeat from {}", machineId);
+        }
     }
 }

web/src/main/resources/application-dev.yml

@@ -1,18 +1,7 @@
-dubbo:
-  registry:
-    address: zookeeper://localhost:2181
-    #address: N/A
 spring:
   thymeleaf:
     cache: false
   datasource:
     url: jdbc:mysql://localhost/reghao_bnt_rdb?useSSL=false&useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2b8
     username: dev
-    password: Dev@123456
-  data:
-    mongodb:
-      host: localhost
-      database: "reghao_spider_rdb"
-      authentication-database: admin
-      username: dev
-      password: Dev@123456
+    password: Dev@123456

web/src/main/resources/application-dev1.yml

@@ -0,0 +1,17 @@
+dubbo:
+  registry:
+    address: zookeeper://localhost:2181
+spring:
+  thymeleaf:
+    cache: false
+  datasource:
+    url: jdbc:mysql://localhost/reghao_bnt_rdb?useSSL=false&useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2b8
+    username: dev
+    password: Dev@123456
+  data:
+    mongodb:
+      host: localhost
+      database: "reghao_spider_rdb"
+      authentication-database: admin
+      username: dev
+      password: Dev@123456

web/src/main/resources/application.yml

@@ -1,9 +1,9 @@
 dubbo:
-  scan:
-    base-packages: cn.reghao.bnt.web.admin.rpc,cn.reghao.bnt.web.console.rpc
   protocol:
     name: dubbo
     port: 14020
+  registry:
+    address: N/A
 server:
   port: 4020
   servlet:
@@ -60,4 +60,4 @@ app:
   cacheDir: /home/reghao/mnt
   baseDir: /opt/data/bntdata
   storeDir: ${app.basedir}/file
-  indexDir: ${app.basedir}/index
+  indexDir: ${app.basedir}/index

web/src/main/resources/application1.yml

@@ -0,0 +1,63 @@
+dubbo:
+  scan:
+    base-packages: cn.reghao.bnt.web.admin.rpc,cn.reghao.bnt.web.console.rpc
+  protocol:
+    name: dubbo
+    port: 14020
+server:
+  port: 4020
+  servlet:
+    session:
+      # 超时时间为一周
+      timeout: 7d
+      persistent: true
+spring:
+  application:
+    name: bnt-web
+  profiles:
+    active: @profile.active@
+  banner:
+    location: classpath:banner.txt
+  servlet:
+    multipart:
+      max-file-size: 1GB
+      max-request-size: 1GB
+  mvc:
+    dispatch-options-request: true
+#    cache: true
+  datasource:
+    driver-class-name: com.mysql.cj.jdbc.Driver
+    type: com.zaxxer.hikari.HikariDataSource
+    hikari:
+      minimum-idle: 5
+      maximum-pool-size: 10
+      auto-commit: true
+      idle-timeout: 30000
+      pool-name: EvaluationHikariCP
+      max-lifetime: 1800000
+      connection-timeout: 30000
+      connection-test-query: SELECT 1
+  jpa:
+    show-sql: false
+    open-in-view: true
+    hibernate:
+      ddl-auto: update
+    properties:
+      hibernate:
+        hbm2ddl:
+          auto: update
+        dialect: org.hibernate.dialect.MySQL57Dialect
+  thymeleaf:
+    prefix: classpath:/templates
+    suffix: .html
+    mode: HTML
+    encoding: utf-8
+    servlet:
+      content-type: text/html
+app:
+  ossEndpoint: https://oss.reghao.cn
+  cookiePath: /home/reghao/Downloads/cookie.txt
+  cacheDir: /home/reghao/mnt
+  baseDir: /opt/data/bntdata
+  storeDir: ${app.basedir}/file
+  indexDir: ${app.basedir}/index