只保留 devops 相关代码

admin-api/pom.xml

@@ -1,19 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>bnt</artifactId>
-        <groupId>cn.reghao.bnt</groupId>
-        <version>1.0.0</version>
-    </parent>
-    <modelVersion>4.0.0</modelVersion>
-
-    <artifactId>admin-api</artifactId>
-    <version>1.0.0-SNAPSHOT</version>
-
-    <properties>
-        <maven.compiler.source>11</maven.compiler.source>
-        <maven.compiler.target>11</maven.compiler.target>
-    </properties>
-
-</project>

admin-api/src/main/java/cn/reghao/bnt/admin/api/dto/AdminMessage.java

@@ -1,21 +0,0 @@
-package cn.reghao.bnt.admin.api.dto;
-
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-
-import java.io.Serializable;
-
-/**
- * @author reghao
- * @date 2024-04-16 17:26:07
- */
-@NoArgsConstructor
-@AllArgsConstructor
-@Getter
-public class AdminMessage implements Serializable {
-    private static final long serialVersionUID = 1L;
-
-    private int type;
-    private String content;
-}

admin-api/src/main/java/cn/reghao/bnt/admin/api/iface/AdminService.java

@@ -1,11 +0,0 @@
-package cn.reghao.bnt.admin.api.iface;
-
-import cn.reghao.bnt.admin.api.dto.AdminMessage;
-
-/**
- * @author reghao
- * @date 2024-04-16 17:20:31
- */
-public interface AdminService {
-    void sendMessage(AdminMessage adminMessage);
-}

admin-api/src/main/java/cn/reghao/bnt/admin/api/iface/SiteConfigService.java

@@ -1,9 +0,0 @@
-package cn.reghao.bnt.admin.api.iface;
-
-/**
- * @author reghao
- * @date 2024-04-06 20:40:56
- */
-public interface SiteConfigService {
-    String getSiteNotice();
-}

browser/dependency-reduced-pom.xml

@@ -1,51 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-  <parent>
-    <artifactId>bnt</artifactId>
-    <groupId>cn.reghao.bnt</groupId>
-    <version>1.0.0</version>
-  </parent>
-  <modelVersion>4.0.0</modelVersion>
-  <artifactId>browser</artifactId>
-  <version>1.0.0-SNAPSHOT</version>
-  <build>
-    <finalName>bnt-browser</finalName>
-    <plugins>
-      <plugin>
-        <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.8.1</version>
-        <configuration>
-          <source>11</source>
-          <target>11</target>
-        </configuration>
-      </plugin>
-      <plugin>
-        <artifactId>maven-shade-plugin</artifactId>
-        <version>3.2.4</version>
-        <executions>
-          <execution>
-            <phase>package</phase>
-            <goals>
-              <goal>shade</goal>
-            </goals>
-            <configuration>
-              <transformers>
-                <transformer>
-                  <mainClass>cn.reghao.bnt.browser.SpiderBrowser</mainClass>
-                </transformer>
-              </transformers>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-    </plugins>
-  </build>
-  <dependencies>
-    <dependency>
-      <groupId>org.projectlombok</groupId>
-      <artifactId>lombok</artifactId>
-      <version>1.18.0</version>
-      <scope>provided</scope>
-    </dependency>
-  </dependencies>
-</project>

browser/pom.xml

@@ -1,65 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>bnt</artifactId>
-        <groupId>cn.reghao.bnt</groupId>
-        <version>1.0.0</version>
-    </parent>
-    <modelVersion>4.0.0</modelVersion>
-
-    <artifactId>browser</artifactId>
-    <version>1.0.0-SNAPSHOT</version>
-
-    <dependencies>
-        <dependency>
-            <groupId>cn.reghao.bnt</groupId>
-            <artifactId>spider</artifactId>
-            <version>1.0.0-SNAPSHOT</version>
-        </dependency>
-
-        <dependency>
-            <groupId>com.squareup.okhttp3</groupId>
-            <artifactId>okhttp</artifactId>
-            <!--<version>4.10.0</version>-->
-            <version>3.14.9</version>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <finalName>bnt-browser</finalName>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-compiler-plugin</artifactId>
-                <version>3.8.1</version>
-                <configuration>
-                    <source>11</source>
-                    <target>11</target>
-                </configuration>
-            </plugin>
-
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-shade-plugin</artifactId>
-                <version>3.2.4</version>
-                <executions>
-                    <execution>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>shade</goal>
-                        </goals>
-                        <configuration>
-                            <transformers>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-                                    <mainClass>cn.reghao.bnt.browser.SpiderBrowser</mainClass>
-                                </transformer>
-                            </transformers>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-</project>

browser/src/main/java/cn/reghao/bnt/browser/SpiderBrowser.java

@@ -1,10 +0,0 @@
-package cn.reghao.bnt.browser;
-
-/**
- * @author reghao
- * @date 2023-09-10 02:24:50
- */
-public class SpiderBrowser {
-    public static void main(String[] args) {
-    }
-}

browser/src/main/java/cn/reghao/bnt/browser/cdp/handler/CdpHandler.java

@@ -1,11 +0,0 @@
-package cn.reghao.bnt.browser.cdp.handler;
-
-import com.google.gson.JsonObject;
-
-/**
- * @author reghao
- * @date 2021-03-15 09:14:51
- */
-public interface CdpHandler {
-    void handle(JsonObject cdpMsg);
-}

browser/src/main/java/cn/reghao/bnt/browser/cdp/handler/event/NetworkEventHandler.java

@@ -1,103 +0,0 @@
-package cn.reghao.bnt.browser.cdp.handler.event;
-
-import cn.reghao.bnt.browser.cdp.handler.CdpHandler;
-import cn.reghao.bnt.browser.cdp.req.NetworkGetResponseBody;
-import cn.reghao.bnt.browser.chrome.PageRequest;
-import cn.reghao.bnt.browser.chrome.ReqInPage;
-import cn.reghao.bnt.browser.chrome.ReqMatcher;
-import cn.reghao.bnt.browser.ws.WebSocketClient;
-import com.google.gson.JsonObject;
-import lombok.extern.slf4j.Slf4j;
-
-import java.util.*;
-
-/**
- * Chrome DevTools Protocol WebSocket 网络事件消息处理器
- *
- * @author reghao
- * @date 2021-03-15 09:08:55
- */
-@Slf4j
-public class NetworkEventHandler implements CdpHandler {
-    private final WebSocketClient wsClient;
-    private final PageRequest pageRequest;
-    private final Map<String, Set<String>> map = new HashMap<>();
-
-    public NetworkEventHandler(WebSocketClient wsClient, PageRequest pageRequest) {
-        this.wsClient = wsClient;
-        this.pageRequest = pageRequest;
-    }
-
-    @Override
-    public void handle(JsonObject cdpMsg) {
-        String method = cdpMsg.get("method").getAsString();
-        JsonObject params = cdpMsg.get("params").getAsJsonObject();
-
-        String requestId = params.get("requestId").getAsString();
-        // Network 事件时序 requestWillBeSent -> responseReceived -> loadingFinished/loadingFailed
-        switch (method) {
-            case "Network.requestWillBeSent":
-                String loaderId = params.get("loaderId").getAsString();
-                JsonObject request = params.get("request").getAsJsonObject();
-                String requestUrl = request.get("url").getAsString();
-                map.computeIfAbsent(loaderId, k -> new HashSet<>());
-                map.get(loaderId).add(requestUrl);
-
-                //log.info("requestUrl -> {}", requestUrl);
-                for (ReqMatcher reqMatcher : pageRequest.getReqMatchers()) {
-                    // 匹配需要的 url 资源
-                    if (requestUrl.contains(reqMatcher.getPattern())) {
-                        ReqInPage reqInPage = new ReqInPage(reqMatcher, requestId, loaderId, requestUrl);
-                        pageRequest.getReqsInNetworkEvent().put(requestId, reqInPage);
-                    }
-                }
-                break;
-            case "Network.loadingFinished":
-                if (pageRequest.getReqsInNetworkEvent().get(requestId) != null) {
-                    // 需要的 url 资源加载完成
-                    getResponseBody(pageRequest.getReqsInNetworkEvent().get(requestId));
-                }
-                break;
-            case "Network.loadingFailed":
-                for (ReqInPage reqInPage : pageRequest.getReqsInNetworkEvent().values()) {
-                    if (requestId.equals(reqInPage.getRequestId())) {
-                        // TODO 应该结束当前页面的请求
-                        log.error("chrome 请求 {} 失败...", reqInPage.getRequestUrl());
-                    }
-                }
-                break;
-            case "Network.responseReceived":
-                break;
-            default:
-                break;
-        }
-    }
-
-    private void getResponseBody(ReqInPage reqInPage) {
-        String requestUrl = reqInPage.getRequestUrl();
-        //String pagePattern = reqInPage.getReqMatcher().getPagePattern();
-        // TODO 临时代码，获取页面 url
-        /*for (Map.Entry<String, Set<String>> entry : map.entrySet()) {
-            String loaderId = entry.getKey();
-            Set<String> urls = entry.getValue();
-
-            if (urls.contains(requestUrl)) {
-                for (String url : urls) {
-                    if (pagePattern != null && url.startsWith(pagePattern)) {
-                        reqInPage.setPageUrl(url);
-                        map.remove(loaderId);
-                        break;
-                    }
-                }
-                break;
-            }
-        }*/
-
-        NetworkGetResponseBody getResponseBody = new NetworkGetResponseBody(reqInPage.getRequestId());
-        int id = getResponseBody.getId();
-        reqInPage.setWsReqId(id);
-        pageRequest.getReqsInGetResponseBody().put(id, reqInPage);
-        // 向 chrome 发送 url 资源可获取事件
-        wsClient.sendMessage(getResponseBody);
-    }
-}

browser/src/main/java/cn/reghao/bnt/browser/cdp/handler/result/GetResponseBodyResultHandler.java

@@ -1,130 +0,0 @@
-package cn.reghao.bnt.browser.cdp.handler.result;
-
-import cn.reghao.bnt.browser.chrome.PageRequest;
-import cn.reghao.bnt.browser.chrome.ReqInPage;
-import cn.reghao.bnt.spider.parser.DataParser;
-import cn.reghao.bnt.spider.url.BodyDataType;
-import cn.reghao.jutil.jdk.serializer.JsonConverter;
-import cn.reghao.bnt.browser.cdp.handler.CdpHandler;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonPrimitive;
-import lombok.extern.slf4j.Slf4j;
-
-import java.nio.charset.StandardCharsets;
-import java.util.*;
-
-/**
- * Chrome DevTools Protocol 获取请求响应 body 的 WebSocket 响应消息处理器
- *
- * @author reghao
- * @date 2021-03-15 09:08:55
- */
-@Slf4j
-public class GetResponseBodyResultHandler implements CdpHandler {
-    private final PageRequest pageRequest;
-
-    public GetResponseBodyResultHandler(PageRequest pageRequest) {
-        this.pageRequest = pageRequest;
-    }
-
-    @Override
-    public void handle(JsonObject cdpMsg) {
-        // 处理 NetworkEventHandler#getResponseBody 中发送的消息
-        int id = cdpMsg.get("id").getAsInt();
-        ReqInPage reqInPage = pageRequest.getReqsInGetResponseBody().get(id);
-        if (reqInPage != null) {
-            if (cdpMsg.get("result") instanceof JsonObject) {
-                handleResponse(reqInPage, cdpMsg.get("result").getAsJsonObject());
-            } else if (cdpMsg.get("error") instanceof JsonObject) {
-                JsonObject error = cdpMsg.get("error").getAsJsonObject();
-                log.error("{} 没有找到 response body", reqInPage.getRequestUrl());
-            }
-        }
-    }
-
-    /**
-     * 获取请求的响应 body
-     *
-     * @param
-     * @return
-     * @date 2021-03-15 上午2:34
-     */
-    private void handleResponse(ReqInPage reqInPage, JsonObject result) {
-        JsonElement bodyElement = result.get("body");
-        if (bodyElement instanceof JsonPrimitive) {
-            String body = bodyElement.getAsString();
-            boolean base64Encoded = result.get("base64Encoded").getAsBoolean();
-            if (base64Encoded) {
-                byte[] bytes = Base64.getUrlDecoder().decode(body);
-                body = new String(bytes, StandardCharsets.UTF_8);
-            }
-
-            if (handle404NotFound(reqInPage.getRequestUrl(), body)) {
-                // 一个页面中，只要有一个需要的资源返回 404，则将所有资源视为 404
-                pageRequest.setNotFound(true);
-            } else {
-                if (!pageRequest.isHandleDirectly()) {
-                    saveResponse(reqInPage, body);
-                } else {
-                    handleResponse(reqInPage, body);
-                }
-            }
-        }
-    }
-
-    /**
-     * 保存响应，返回使用
-     *
-     * @param
-     * @return
-     * @date 2021-08-05 下午9:36
-     */
-    private void saveResponse(ReqInPage reqInPage, String body) {
-        String dataType = reqInPage.getReqMatcher().getDataType();
-        if (dataType.equals(BodyDataType.m3u8.name()) && !body.contains("#EXTINF")) {
-            return;
-        }
-
-        reqInPage.setBody(body);
-        pageRequest.getReqsInGetResponseBodyDone().add(reqInPage);
-    }
-
-    /**
-     * 直接处理响应
-     *
-     * @param
-     * @return
-     * @date 2021-08-05 下午9:36
-     */
-    private synchronized void handleResponse(ReqInPage reqInPage, String body) {
-        String pageUrl = reqInPage.getRequestUrl();
-        DataParser dataParser = reqInPage.getReqMatcher().getDataParser();
-        try {
-            dataParser.parse(pageUrl, body);
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-    }
-
-    /**
-     * 处理 404
-     *
-     * @param
-     * @return
-     * @date 2021-03-19 下午2:40
-     */
-    private boolean handle404NotFound(String requestUrl, String body) {
-        try {
-            JsonElement errors = JsonConverter.jsonToJsonElement(body).getAsJsonObject().get("errors");
-            if (errors instanceof JsonArray) {
-                log.error("{} 不存在 -> {}", requestUrl, body);
-                return true;
-            }
-        } catch (Exception e) {
-            //
-        }
-        return false;
-    }
-}

browser/src/main/java/cn/reghao/bnt/browser/cdp/handler/result/PageNavigateResultHandler.java

@@ -1,22 +0,0 @@
-package cn.reghao.bnt.browser.cdp.handler.result;
-
-import cn.reghao.bnt.browser.cdp.handler.CdpHandler;
-import com.google.gson.JsonObject;
-import lombok.extern.slf4j.Slf4j;
-
-/**
- * Chrome DevTools Protocol 页面请求 WebSocket 响应消息处理器
- *
- * @author reghao
- * @date 2021-03-15 09:08:55
- */
-@Slf4j
-public class PageNavigateResultHandler implements CdpHandler {
-    @Override
-    public void handle(JsonObject cdpMsg) {
-        int id = cdpMsg.get("id").getAsInt();
-        JsonObject result = cdpMsg.get("result").getAsJsonObject();
-        String frameId = result.get("frameId").getAsString();
-        String loaderId = result.get("loaderId").getAsString();
-    }
-}

browser/src/main/java/cn/reghao/bnt/browser/cdp/msg/CdpMethodResultMessage.java

@@ -1,44 +0,0 @@
-package cn.reghao.bnt.browser.cdp.msg;
-
-import cn.reghao.bnt.browser.chrome.PageRequest;
-import cn.reghao.bnt.browser.cdp.handler.result.GetResponseBodyResultHandler;
-import cn.reghao.bnt.browser.cdp.handler.result.PageNavigateResultHandler;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import lombok.extern.slf4j.Slf4j;
-
-/**
- * Chrome DevTools Protocol 方法调用 WebSocket 响应消息
- *
- * @author reghao
- * @date 2021-03-15 09:08:55
- */
-@Slf4j
-public class CdpMethodResultMessage {
-    private PageNavigateResultHandler pageNavigateResultHandler;
-    private GetResponseBodyResultHandler getResponseBodyResultHandler;
-
-    public CdpMethodResultMessage(PageRequest pageRequest) {
-        this.pageNavigateResultHandler = new PageNavigateResultHandler();
-        this.getResponseBodyResultHandler = new GetResponseBodyResultHandler(pageRequest);
-    }
-
-    public void parse(JsonObject cdpMsg) {
-        if (cdpMsg.get("result") != null) {
-            JsonObject result = cdpMsg.get("result").getAsJsonObject();
-
-            JsonElement frameIdElement = result.get("frameId");
-            if (frameIdElement != null) {
-                pageNavigateResultHandler.handle(cdpMsg);
-            }
-
-            JsonElement bodyElement = result.get("body");
-            if (bodyElement != null) {
-                getResponseBodyResultHandler.handle(cdpMsg);
-            }
-        } else if (cdpMsg.get("error") != null) {
-            JsonObject error = cdpMsg.get("error").getAsJsonObject();
-            log.error("请求错误...");
-        }
-    }
-}

browser/src/main/java/cn/reghao/bnt/browser/cdp/msg/CdpNetworkEventMessage.java

@@ -1,26 +0,0 @@
-package cn.reghao.bnt.browser.cdp.msg;
-
-import cn.reghao.bnt.browser.chrome.PageRequest;
-import cn.reghao.bnt.browser.cdp.handler.event.NetworkEventHandler;
-import cn.reghao.bnt.browser.ws.WebSocketClient;
-import com.google.gson.JsonObject;
-import lombok.extern.slf4j.Slf4j;
-
-/**
- * Chrome DevTools Protocol WebSocket 网络事件消息
- *
- * @author reghao
- * @date 2021-03-15 09:08:55
- */
-@Slf4j
-public class CdpNetworkEventMessage {
-    private NetworkEventHandler networkEventHandler;
-
-    public CdpNetworkEventMessage(WebSocketClient wsClient, PageRequest pageRequest) {
-        this.networkEventHandler = new NetworkEventHandler(wsClient, pageRequest);
-    }
-
-    public void parse(JsonObject cdpMsg) {
-        networkEventHandler.handle(cdpMsg);
-    }
-}

browser/src/main/java/cn/reghao/bnt/browser/cdp/req/CdpReq.java

@@ -1,26 +0,0 @@
-package cn.reghao.bnt.browser.cdp.req;
-
-import java.util.concurrent.ThreadLocalRandom;
-
-/**
- * @author reghao
- * @date 2021-03-14 01:56:32
- */
-public class CdpReq {
-    private int id;
-    private String method;
-
-    public CdpReq(String method) {
-        this.id = randomInt();
-        this.method = method;
-    }
-
-    public int getId() {
-        return this.id;
-    }
-
-    protected int randomInt() {
-        int min = 10000, max = 99999;
-        return ThreadLocalRandom.current().nextInt(min, max + 1);
-    }
-}

browser/src/main/java/cn/reghao/bnt/browser/cdp/req/NetworkEnable.java

@@ -1,24 +0,0 @@
-package cn.reghao.bnt.browser.cdp.req;
-
-/**
- * @author reghao
- * @date 2021-03-14 04:26:04
- */
-public class NetworkEnable extends CdpReq {
-    private Params params;
-
-    public NetworkEnable() {
-        super("Network.enable");
-        this.params = new Params();
-    }
-
-    static class Params {
-        private long maxResourceBufferSize;
-        private long maxTotalBufferSize;
-
-        public Params() {
-            this.maxResourceBufferSize = 1024*1024*100;
-            this.maxTotalBufferSize = 1024*1024*200;
-        }
-    }
-}

browser/src/main/java/cn/reghao/bnt/browser/cdp/req/NetworkGetCookies.java

@@ -1,26 +0,0 @@
-package cn.reghao.bnt.browser.cdp.req;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * @author reghao
- * @date 2021-03-14 19:23:26
- */
-public class NetworkGetCookies extends CdpReq {
-    private Params params;
-
-    public NetworkGetCookies(String url) {
-        super("Network.getCookies");
-        this.params = new Params(url);
-    }
-
-    static class Params {
-        private List<String> urls;
-        public Params(String url) {
-            List<String> list = new ArrayList<>();
-            list.add(url);
-            this.urls = list;
-        }
-    }
-}

browser/src/main/java/cn/reghao/bnt/browser/cdp/req/NetworkGetResponseBody.java

@@ -1,22 +0,0 @@
-package cn.reghao.bnt.browser.cdp.req;
-
-/**
- * @author reghao
- * @date 2021-03-14 19:23:26
- */
-public class NetworkGetResponseBody extends CdpReq {
-    private Params params;
-
-    public NetworkGetResponseBody(String requestId) {
-        super("Network.getResponseBody");
-        this.params = new Params(requestId);
-    }
-
-    static class Params {
-        private String requestId;
-
-        public Params(String requestId) {
-            this.requestId = requestId;
-        }
-    }
-}

browser/src/main/java/cn/reghao/bnt/browser/cdp/req/PageNavigate.java

@@ -1,23 +0,0 @@
-package cn.reghao.bnt.browser.cdp.req;
-
-/**
- * 请求页面
- *
- * @author reghao
- * @date 2021-03-14 19:23:26
- */
-public class PageNavigate extends CdpReq {
-    private Params params;
-
-    public PageNavigate(String url) {
-        super("Page.navigate");
-        this.params = new Params(url);
-    }
-
-    static class Params {
-        private String url;
-        public Params(String url) {
-            this.url = url;
-        }
-    }
-}

browser/src/main/java/cn/reghao/bnt/browser/cdp/req/PageSetDownloadBehavior.java

@@ -1,26 +0,0 @@
-package cn.reghao.bnt.browser.cdp.req;
-
-/**
- * 文件下载设置
- *
- * @author reghao
- * @date 2022-06-02 10:25:21
- */
-public class PageSetDownloadBehavior extends CdpReq {
-    private Params params;
-
-    public PageSetDownloadBehavior(String dir) {
-        super("Page.setDownloadBehavior");
-        this.params = new PageSetDownloadBehavior.Params(dir);
-    }
-
-    static class Params {
-        private String behavior;
-        private String downloadPath;
-
-        public Params(String downloadPath) {
-            this.behavior = "allowAndName";
-            this.downloadPath = downloadPath;
-        }
-    }
-}

browser/src/main/java/cn/reghao/bnt/browser/cdp/req/RemoteTarget.java

@@ -1,21 +0,0 @@
-package cn.reghao.bnt.browser.cdp.req;
-
-import lombok.Data;
-
-/**
- * http://localhost:9999/json 返回的数据格式
- *
- * @author reghao
- * @date 2021-03-14 23:42:18
- */
-@Data
-public class RemoteTarget {
-    private String description;
-    private String devtoolsFrontendUrl;
-    // tabId
-    private String id;
-    private String title;
-    private String type;
-    private String url;
-    private String webSocketDebuggerUrl;
-}

browser/src/main/java/cn/reghao/bnt/browser/cdp/req/TargetCreateTarget.java

@@ -1,23 +0,0 @@
-package cn.reghao.bnt.browser.cdp.req;
-
-/**
- * 打开新标签
- *
- * @author reghao
- * @date 2021-08-07 14:15:40
- */
-public class TargetCreateTarget extends CdpReq {
-    private Params params;
-
-    public TargetCreateTarget(String url) {
-        super("Target.createTarget");
-        this.params = new Params(url);
-    }
-
-    static class Params {
-        private String url;
-        public Params(String url) {
-            this.url = url;
-        }
-    }
-}

browser/src/main/java/cn/reghao/bnt/browser/chrome/AbstractChrome.java

@@ -1,298 +0,0 @@
-package cn.reghao.bnt.browser.chrome;
-
-import cn.reghao.bnt.browser.cdp.req.*;
-import cn.reghao.bnt.browser.ws.WebSocketClient;
-import cn.reghao.bnt.browser.cdp.req.*;
-import cn.reghao.jutil.jdk.serializer.JsonConverter;
-import lombok.extern.slf4j.Slf4j;
-import okhttp3.Call;
-import okhttp3.OkHttpClient;
-import okhttp3.Request;
-import okhttp3.Response;
-
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.stream.Collectors;
-
-/**
- * Chrome 浏览器抽象类
- *
- * @author reghao
- * @date 2021-03-19 13:58:07
- */
-@Slf4j
-public abstract class AbstractChrome {
-    protected boolean isHeadless;
-    protected boolean enableProxy;
-    protected int reqTimeout;
-    protected int remotePort;
-    protected String tabsApi;
-    protected String wsPrefix;
-    protected Map<String, PageRequest> pageRequestMap = new ConcurrentHashMap<>();
-    protected Map<String, WebSocketClient> wsClientMap = new ConcurrentHashMap<>();
-
-    public AbstractChrome(boolean isHeadless, boolean enableProxy) {
-        this.isHeadless = isHeadless;
-        this.enableProxy = enableProxy;
-        // 需要代理的页面超时 30s
-        this.reqTimeout = enableProxy ? 30 : 10;
-        init();
-    }
-
-    private void init() {
-        this.remotePort = randomPort();
-        this.tabsApi = String.format("http://localhost:%s/json", remotePort);
-        this.wsPrefix = String.format("ws://localhost:%s/devtools/page/", remotePort);
-    }
-
-    private int randomPort() {
-        int min = 10000, max = 20000;
-        return ThreadLocalRandom.current().nextInt(min, max + 1);
-    }
-
-    protected void enableWebSocket() {
-        Set<String> tabIds = tabIds();
-        if (tabIds.size() != 1) {
-            log.error("Chrome 启动时默认打开的 tab 不是 1 个，而是 {} 个, 启用 WebSocket 连接失败...", tabIds.size());
-            return;
-        }
-
-        String tabId = tabIds.iterator().next();
-        PageRequest pageRequest = new PageRequest();
-        pageRequestMap.putIfAbsent(tabId, pageRequest);
-        wsClientMap.putIfAbsent(tabId, new WebSocketClient(wsPrefix + tabId, pageRequest));
-    }
-
-    private Set<String> tabIds() {
-        OkHttpClient okHttpClient = new OkHttpClient();
-        Request request = new Request.Builder()
-                .url(tabsApi)
-                .get()
-                .build();
-
-        Call call = okHttpClient.newCall(request);
-        try {
-            Response response = call.execute();
-            assert response.body() != null;
-            String body = response.body().string();
-            return JsonConverter.jsonToObjects(body, RemoteTarget.class).stream()
-                    .filter(remoteTarget -> remoteTarget.getType().equals("page"))
-                    .filter(remoteTarget -> {
-                        String url = remoteTarget.getUrl();
-                        return url.equals("data:,") || url.equals("chrome://newtab/");
-                    })
-                    .map(RemoteTarget::getId)
-                    .collect(Collectors.toSet());
-        } catch (Exception e) {
-            log.error("获取 chrome tab 列表失败 -> {}", e.getMessage());
-        }
-        return Collections.emptySet();
-    }
-
-    protected void openTabs(int tabNum) {
-        if (wsClientMap.size() != 1) {
-            log.error("打开新 tab 失败");
-            return;
-        }
-        WebSocketClient wsClient = wsClientMap.entrySet().iterator().next().getValue();
-        for (int i = 1; i < tabNum; i++) {
-            TargetCreateTarget targetCreateTarget = new TargetCreateTarget("");
-            wsClient.sendMessage(targetCreateTarget);
-        }
-
-        Set<String> tabIds = tabIds();
-        if (tabIds.size() != tabNum) {
-            log.error("打开新 tab 失败");
-            return;
-        }
-        tabIds.forEach(this::enableTabNetworkDebug);
-    }
-
-    /**
-     * 启用 tab 网络 debug
-     *
-     * @param
-     * @return
-     * @date 2021-08-07 下午3:40
-     */
-    private void enableTabNetworkDebug(String tabId) {
-        PageRequest pageRequest = pageRequestMap.get(tabId);
-        if (pageRequest == null) {
-            pageRequest = new PageRequest();
-            pageRequestMap.put(tabId, pageRequest);
-        }
-
-        NetworkEnable networkEnable = new NetworkEnable();
-        WebSocketClient wsClient = wsClientMap.get(tabId);
-        if (wsClient == null) {
-            wsClientMap.put(tabId, new WebSocketClient(wsPrefix + tabId, pageRequest));
-        } else {
-            wsClient.sendMessage(networkEnable);
-        }
-    }
-
-    public List<String> availTabIds() {
-        return new ArrayList<>(wsClientMap.keySet());
-    }
-
-    public List<ReqInPage> getDynamicPage(String tabId, String url, boolean autoScroll, Set<ReqMatcher> reqMatchers) {
-        PageRequest pageRequest = pageRequestMap.get(tabId);
-        pageRequest.setReqMatchers(reqMatchers);
-        PageNavigate pageNavigate = new PageNavigate(url);
-        int id = pageNavigate.getId();
-        // 向 chrome 发送打开页面事件
-        wsClientMap.get(tabId).sendMessage(pageNavigate);
-        for (int i = 0; i < reqTimeout; i++) {
-            try {
-                Thread.sleep(1_000);
-                //log.info("请求已用时 {}s...", i+1);
-            } catch (InterruptedException e) {
-                e.printStackTrace();
-            }
-
-            if (pageRequest.isNotFound()) {
-                log.info("{} 中的资源不存在，请开始下一次请求...", url);
-                pageRequest.clearAll();
-                return null;
-            } else if (pageRequest.isLoadDone()) {
-                log.info("请求处理完成，请开始下一次请求...");
-                if (autoScroll) {
-                    autoScrollPage();
-                }
-                return pageRequest.reqsInPage();
-            }
-        }
-
-        pageRequest.clearAll();
-        return Collections.emptyList();
-    }
-
-    public List<ReqInPage> getDynamicPage(String url, boolean autoScroll, Set<ReqMatcher> reqMatchers) {
-        Map.Entry<String, PageRequest> entry = pageRequestMap.entrySet().iterator().next();
-        String tabId = entry.getKey();
-        PageRequest pageRequest = entry.getValue();
-        pageRequest.setReqMatchers(reqMatchers);
-        pageRequest.setHandleDirectly(true);
-
-        PageNavigate pageNavigate = new PageNavigate(url);
-        int id = pageNavigate.getId();
-        wsClientMap.get(tabId).sendMessage(pageNavigate);
-        for (int i = 0; i < reqTimeout; i++) {
-            try {
-                Thread.sleep(1_000);
-                //log.info("请求已用时 {}s...", i+1);
-            } catch (InterruptedException e) {
-                e.printStackTrace();
-            }
-
-            if (pageRequest.isNotFound()) {
-                log.info("{} 中的资源不存在，请开始下一次请求...", url);
-                pageRequest.clearAll();
-                return null;
-            } else if (pageRequest.isLoadDone()) {
-                log.info("请求处理完成，请开始下一次请求...");
-                if (autoScroll) {
-                    autoScrollPage();
-                }
-                return pageRequest.reqsInPage();
-            }
-        }
-
-        pageRequest.clearAll();
-        return Collections.emptyList();
-    }
-
-    public void getAndHandleDynamicPage(String url, Set<ReqMatcher> reqMatchers) {
-        Map.Entry<String, PageRequest> entry = pageRequestMap.entrySet().iterator().next();
-        String tabId = entry.getKey();
-        PageRequest pageRequest = entry.getValue();
-        pageRequest.setReqMatchers(reqMatchers);
-        pageRequest.setHandleDirectly(true);
-
-        PageNavigate pageNavigate = new PageNavigate(url);
-        int id = pageNavigate.getId();
-        wsClientMap.get(tabId).sendMessage(pageNavigate);
-        int timeout = 3600*24;
-        for (int i = 0; i < timeout; i++) {
-            try {
-                Thread.sleep(1_000);
-            } catch (InterruptedException e) {
-                e.printStackTrace();
-            }
-        }
-    }
-
-    public void getAndHandleDynamicPageAuto(String url, Set<ReqMatcher> reqMatchers) {
-        Map.Entry<String, PageRequest> entry = pageRequestMap.entrySet().iterator().next();
-        String tabId = entry.getKey();
-        PageRequest pageRequest = entry.getValue();
-        pageRequest.setReqMatchers(reqMatchers);
-        pageRequest.setHandleDirectly(true);
-
-        PageNavigate pageNavigate = new PageNavigate(url);
-        int id = pageNavigate.getId();
-        wsClientMap.get(tabId).sendMessage(pageNavigate);
-
-        try {
-            log.info("休眠 10s 后请求下一个页面");
-            Thread.sleep(10_000);
-        } catch (InterruptedException e) {
-            e.printStackTrace();
-        }
-    }
-
-    @Deprecated
-    public void getAndHandleDynamicPages(Set<String> urls, Set<ReqMatcher> reqMatchers) {
-        Map.Entry<String, PageRequest> entry = pageRequestMap.entrySet().iterator().next();
-        String tabId = entry.getKey();
-        PageRequest pageRequest = entry.getValue();
-        pageRequest.setReqMatchers(reqMatchers);
-        pageRequest.setHandleDirectly(true);
-        for (String url : urls) {
-            PageNavigate pageNavigate = new PageNavigate(url);
-            int id = pageNavigate.getId();
-            wsClientMap.get(tabId).sendMessage(pageNavigate);
-
-            try {
-                log.info("休眠 3s 后请求下一个页面");
-                Thread.sleep(3_000);
-            } catch (InterruptedException e) {
-                e.printStackTrace();
-            }
-        }
-    }
-
-    public void download(Set<String> urls, String dir) {
-        Map.Entry<String, PageRequest> entry = pageRequestMap.entrySet().iterator().next();
-        String tabId = entry.getKey();
-        PageRequest pageRequest = entry.getValue();
-        pageRequest.setReqMatchers(new HashSet<>());
-        pageRequest.setHandleDirectly(true);
-
-        WebSocketClient wsClient = wsClientMap.get(tabId);
-        PageSetDownloadBehavior downloadBehavior = new PageSetDownloadBehavior(dir);
-        wsClient.sendMessage(downloadBehavior);
-        for (String url : urls) {
-            PageNavigate pageNavigate = new PageNavigate(url);
-            int id = pageNavigate.getId();
-            wsClient.sendMessage(pageNavigate);
-
-            try {
-                log.info("休眠 3s 后请求下一个页面");
-                Thread.sleep(3_000);
-            } catch (InterruptedException e) {
-                e.printStackTrace();
-            }
-        }
-    }
-
-    /**
-     * 自动滚动页面
-     *
-     * @param
-     * @return
-     * @date 2021-08-06 上午9:54
-     */
-    abstract void autoScrollPage();
-}

browser/src/main/java/cn/reghao/bnt/browser/chrome/ChromeBrowser.java

@@ -1,71 +0,0 @@
-package cn.reghao.bnt.browser.chrome;
-
-import cn.reghao.bnt.browser.cdp.req.NetworkGetCookies;
-import lombok.extern.slf4j.Slf4j;
-import java.util.*;
-
-/**
- * @author reghao
- * @date 2021-03-15 00:59:58
- */
-@Slf4j
-public class ChromeBrowser extends AbstractChrome {
-    public ChromeBrowser(int tabNum, boolean isHeadless, boolean enableProxy) {
-        super(isHeadless, enableProxy);
-        init(tabNum);
-    }
-
-    public ChromeBrowser(boolean isHeadless, boolean enableProxy) {
-        super(isHeadless, enableProxy);
-        init(1);
-    }
-
-    private void init(int tabNum) {
-        ProcessBuilder pb = new ProcessBuilder(chromeArguments());
-        try {
-            pb.start();
-            log.info("休眠 5s 等待 Chrome 完全启动...");
-            Thread.sleep(5_000);
-            enableWebSocket();
-            openTabs(tabNum);
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-    }
-
-    private List<String> chromeArguments() {
-        List<String> cmd = new ArrayList<>();
-        //cmd.add("/usr/bin/google-chrome-unstable");
-        cmd.add("/usr/bin/google-chrome-stable");
-        if (isHeadless) {
-            cmd.add("--headless");
-        }
-        cmd.add("--disable-gpu");
-        cmd.add("--user-data-dir=");
-        cmd.add("--remote-debugging-port=" + remotePort);
-        if (enableProxy) {
-            cmd.add("--proxy-server=socks://127.0.0.1:1080");
-        }
-        return cmd;
-    }
-
-    public String getCookies(String url) {
-        NetworkGetCookies getCookies = new NetworkGetCookies(url);
-        log.info("获取 cookies 的 id -> " + getCookies.getId());
-        //wsClient.sendMessage(getCookies);
-        return null;
-    }
-
-    public void setCookies() {
-    }
-
-    @Override
-    public void autoScrollPage() {
-        log.info("chrome 浏览器休眠 60s 模拟页面滚动...");
-        try {
-            Thread.sleep(60_000);
-        } catch (InterruptedException e) {
-            e.printStackTrace();
-        }
-    }
-}

browser/src/main/java/cn/reghao/bnt/browser/chrome/PageRequest.java

@@ -1,79 +0,0 @@
-package cn.reghao.bnt.browser.chrome;
-
-import lombok.Data;
-
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.stream.Collectors;
-
-/**
- * 页面请求，包含若干动态请求(和浏览器的 tab 关联)
- *
- * @author reghao
- * @date 2021-03-19 20:49:56
- */
-@Data
-public class PageRequest {
-    // 是否直接处理页面中的响应
-    private boolean isHandleDirectly;
-    private boolean notFound;
-    // 页面中需要匹配的请求
-    private Set<ReqMatcher> reqMatchers;
-    // 匹配到的请求
-    private Map<String, ReqInPage> reqsInNetworkEvent;
-    // 需要从 chrome 中获取响应的请求
-    private Map<Integer, ReqInPage> reqsInGetResponseBody;
-    // 已经从 chrome 中获取到响应的请求
-    private List<ReqInPage> reqsInGetResponseBodyDone;
-
-    public PageRequest() {
-        this.isHandleDirectly = false;
-        this.notFound = false;
-        this.reqsInNetworkEvent = new ConcurrentHashMap<>();
-        this.reqsInGetResponseBody = new ConcurrentHashMap<>();
-        this.reqsInGetResponseBodyDone = new CopyOnWriteArrayList<>();
-    }
-
-    /**
-     * 页面是否加载完成
-     *
-     * @param
-     * @return
-     * @date 2021-08-06 上午9:51
-     */
-    public boolean isLoadDone() {
-        Set<ReqMatcher> set = reqsInGetResponseBodyDone.stream()
-                .map(ReqInPage::getReqMatcher).collect(Collectors.toSet());
-        // TODO 判断 set 和 reqMatchers 中的元素是否相同
-        return reqMatchers.size() == set.size();
-    }
-
-    /**
-     * 页面中需要匹配的请求，每次请求页面前调用
-     *
-     * @param
-     * @return
-     * @date 2021-08-07 下午1:50
-     */
-    public synchronized void setReqMatchers(Set<ReqMatcher> set) {
-        reqMatchers = set;
-    }
-
-    public List<ReqInPage> reqsInPage() {
-        List<ReqInPage> list = new ArrayList<>(reqsInGetResponseBodyDone);
-        clearAll();
-        return list;
-    }
-
-    public void clearAll() {
-        this.notFound = false;
-        clearCache();
-    }
-
-    private void clearCache() {
-        reqsInGetResponseBodyDone.clear();
-        reqsInGetResponseBody.clear();
-        reqsInNetworkEvent.clear();
-    }
-}

browser/src/main/java/cn/reghao/bnt/browser/chrome/ReqInPage.java

@@ -1,31 +0,0 @@
-package cn.reghao.bnt.browser.chrome;
-
-import lombok.Data;
-
-/**
- * 页面中的动态请求，URL 在页面中唯一
- *
- * @author reghao
- * @date 2021-03-12 23:34:56
- */
-@Data
-public class ReqInPage {
-    // WebSocket request id(global unique)
-    private int wsReqId;
-    private ReqMatcher reqMatcher;
-    // CDP Network 事件中的 requestId
-    private String requestId;
-    // 同一个页面加载的所有请求的 loaderId 都相同
-    private String loaderId;
-    // pattern 匹配的 URL
-    private String requestUrl;
-    private String body;
-    //private String pageUrl;
-
-    public ReqInPage(ReqMatcher reqMatcher, String requestId, String loaderId, String requestUrl) {
-        this.reqMatcher = reqMatcher;
-        this.requestId = requestId;
-        this.loaderId = loaderId;
-        this.requestUrl = requestUrl;
-    }
-}

browser/src/main/java/cn/reghao/bnt/browser/chrome/ReqMatcher.java

@@ -1,37 +0,0 @@
-package cn.reghao.bnt.browser.chrome;
-
-import cn.reghao.bnt.spider.parser.DataParser;
-import cn.reghao.bnt.spider.url.BodyDataType;
-import cn.reghao.bnt.spider.url.Site;
-import lombok.Data;
-
-/**
- * 匹配页面中的特定请求
- *
- * @author reghao
- * @date 2021-08-02 18:15:11
- */
-@Data
-public class ReqMatcher {
-    private String site;
-    // 能唯一确定一个请求的字符串匹配模板
-    private String pattern;
-    private String dataType;
-    private DataParser dataParser;
-    //private String pagePattern;
-
-    public ReqMatcher(Site site, String pattern, BodyDataType bodyDataType, DataParser dataParser) {
-        this.site = site.name();
-        this.pattern = pattern;
-        this.dataType = bodyDataType.name();
-        this.dataParser = dataParser;
-    }
-
-    /*public ReqMatcher(Site site, String pattern, BodyDataType bodyDataType, DataParser dataParser, String pagePattern) {
-        this.site = site.name();
-        this.pattern = pattern;
-        this.dataType = bodyDataType.name();
-        this.dataParser = dataParser;
-        this.pagePattern = pagePattern;
-    }*/
-}

browser/src/main/java/cn/reghao/bnt/browser/parser/ChromeDataParser.java

@@ -1,54 +0,0 @@
-package cn.reghao.bnt.browser.parser;
-
-import cn.reghao.jutil.jdk.http.WebRequest;
-import cn.reghao.jutil.jdk.http.WebResponse;
-import cn.reghao.jutil.jdk.result.WebResult;
-import cn.reghao.jutil.jdk.serializer.JsonConverter;
-import cn.reghao.jutil.tool.http.DefaultWebRequest;
-import cn.reghao.bnt.browser.SpiderBrowser;
-import cn.reghao.bnt.spider.parser.DataParser;
-import cn.reghao.bnt.spider.url.CrawlUrl;
-import cn.reghao.bnt.spider.url.RawData;
-import cn.reghao.bnt.spider.url.Site;
-import com.google.gson.reflect.TypeToken;
-import lombok.extern.slf4j.Slf4j;
-
-import java.lang.reflect.Type;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * @author reghao
- * @date 2023-09-10 02:22:39
- */
-@Slf4j
-public class ChromeDataParser implements DataParser {
-    private final WebRequest webRequest = new DefaultWebRequest();
-    private final String api = "http://spider.reghao.cn/api/crawler/rawdata";
-    private Map<String, String> map = new HashMap<>();
-
-    @Override
-    public Map<String, Object> parse(String url, String body) throws InterruptedException {
-        map.forEach((key, value) -> {
-            if (url.contains(key)) {
-                CrawlUrl crawlUrl = new CrawlUrl(Site.taobao.name(), value, url);
-                RawData rawData = new RawData(crawlUrl, body);
-                WebResponse webResponse = webRequest.postJson(api, JsonConverter.objectToJson(rawData));
-                int statusCode = webResponse.getStatusCode();
-                if (statusCode != 200) {
-                    return;
-                }
-
-                String body1 = webResponse.getBody();
-                Type type = new TypeToken<WebResult<Boolean>>(){}.getType();
-                WebResult<Boolean> webResult = JsonConverter.jsonToObject(body1, type);
-                if (webResult.getCode() != 0) {
-                    log.error("请求提交失败: {}", webResult.getMsg());
-                }
-            }
-        });
-
-        return Collections.emptyMap();
-    }
-}

browser/src/main/java/cn/reghao/bnt/browser/ws/CdpMessageHandler.java

@@ -1,48 +0,0 @@
-package cn.reghao.bnt.browser.ws;
-
-import cn.reghao.bnt.browser.cdp.msg.CdpMethodResultMessage;
-import cn.reghao.bnt.browser.cdp.msg.CdpNetworkEventMessage;
-import cn.reghao.jutil.jdk.serializer.JsonConverter;
-import cn.reghao.bnt.browser.chrome.PageRequest;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import lombok.extern.slf4j.Slf4j;
-
-/**
- * Chrome DevTools Protocol WebSocket 消息处理器
- *
- * @author reghao
- * @date 2021-03-14 20:26:58
- */
-@Slf4j
-public class CdpMessageHandler implements MessageHandler {
-    private CdpMethodResultMessage methodResultMessage;
-    private CdpNetworkEventMessage networkEventMessage;
-
-    public CdpMessageHandler(WebSocketClient wsClient, PageRequest pageRequest) {
-        this.methodResultMessage = new CdpMethodResultMessage(pageRequest);
-        this.networkEventMessage = new CdpNetworkEventMessage(wsClient, pageRequest);
-    }
-
-    @Override
-    public void handleMessage(String message) {
-        JsonElement jsonElement = JsonConverter.jsonToJsonElement(message);
-        if (jsonElement instanceof JsonObject) {
-            JsonObject cdpMsg = jsonElement.getAsJsonObject();
-
-            if (cdpMsg.get("result") != null) {
-                //log.info("接收到 Chrome WebSocket 响应消息");
-                methodResultMessage.parse(cdpMsg);
-            }
-
-            JsonElement methodElement = cdpMsg.get("method");
-            if (methodElement != null) {
-                String method = methodElement.getAsString();
-                if (method.startsWith("Network.")) {
-                    //log.info("接收到 Network 事件消息 -> {}", message);
-                    networkEventMessage.parse(cdpMsg);
-                }
-            }
-        }
-    }
-}

browser/src/main/java/cn/reghao/bnt/browser/ws/MessageHandler.java

@@ -1,9 +0,0 @@
-package cn.reghao.bnt.browser.ws;
-
-/**
- * @author reghao
- * @date 2021-03-14 20:26:39
- */
-public interface MessageHandler {
-    void handleMessage(String message);
-}

browser/src/main/java/cn/reghao/bnt/browser/ws/WebSocketClient.java

@@ -1,37 +0,0 @@
-package cn.reghao.bnt.browser.ws;
-
-import cn.reghao.jutil.jdk.serializer.JsonConverter;
-import cn.reghao.bnt.browser.chrome.PageRequest;
-import lombok.extern.slf4j.Slf4j;
-import okhttp3.*;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * @author reghao
- * @date 2021-03-14 18:54:14
- */
-@Slf4j
-public class WebSocketClient {
-    private final WebSocket webSocket;
-
-    public WebSocketClient(String ws, PageRequest pageRequest) {
-        Request request = new Request.Builder().get().url(ws).build();
-        OkHttpClient okHttpClient = new OkHttpClient.Builder()
-                .readTimeout(5, TimeUnit.SECONDS)//设置读取超时时间
-                .writeTimeout(5, TimeUnit.SECONDS)//设置写的超时时间
-                .connectTimeout(5, TimeUnit.SECONDS)//设置连接超时时间
-                .build();
-
-        this.webSocket = okHttpClient.newWebSocket(request,
-                new WebSocketListenerImpl(new CdpMessageHandler(this, pageRequest)));
-    }
-
-    public void sendMessage(String message) {
-        webSocket.send(message);
-    }
-
-    public void sendMessage(Object object) {
-        webSocket.send(JsonConverter.objectToJson(object));
-    }
-}

browser/src/main/java/cn/reghao/bnt/browser/ws/WebSocketListenerImpl.java

@@ -1,55 +0,0 @@
-package cn.reghao.bnt.browser.ws;
-
-import lombok.extern.slf4j.Slf4j;
-import okhttp3.Response;
-import okhttp3.WebSocket;
-import okhttp3.WebSocketListener;
-import okio.ByteString;
-
-/**
- * @author reghao
- * @date 2021-03-14 20:18:24
- */
-@Slf4j
-public class WebSocketListenerImpl extends WebSocketListener {
-    private final MessageHandler messageHandler;
-
-    public WebSocketListenerImpl(MessageHandler messageHandler) {
-        this.messageHandler = messageHandler;
-    }
-
-    @Override
-    public void onOpen(WebSocket webSocket, Response response) {
-        super.onOpen(webSocket, response);
-        log.info("WebSocket 连接建立...");
-    }
-
-    @Override
-    public void onClosed(WebSocket webSocket, int code, String reason) {
-        super.onClosed(webSocket, code, reason);
-        log.info("WebSocket 连接断开...");
-    }
-
-    @Override
-    public void onFailure(WebSocket webSocket, Throwable throwable, Response response) {
-        log.info("WebSocket 连接失败 -> {} - {}", throwable.toString(), response.toString());
-        super.onFailure(webSocket, throwable, response);
-    }
-
-    @Override
-    public void onMessage(WebSocket webSocket, String text) {
-        super.onMessage(webSocket, text);
-        try {
-            messageHandler.handleMessage(text);
-        } catch (Exception e) {
-            log.error("处理消息时发生错误 -> {}", e.getMessage());
-            e.printStackTrace();
-        }
-    }
-
-    @Override
-    public void onMessage(WebSocket webSocket, ByteString bytes) {
-        super.onMessage(webSocket, bytes);
-        log.info("接收到服务端字节消息");
-    }
-}

browser/src/test/java/cn/reghao/bnt/browser/TmallSpiderTest.java

@@ -1,46 +0,0 @@
-package cn.reghao.bnt.browser;
-
-import cn.reghao.bnt.browser.chrome.AbstractChrome;
-import cn.reghao.bnt.browser.chrome.ChromeBrowser;
-import cn.reghao.bnt.browser.chrome.ReqMatcher;
-import cn.reghao.bnt.browser.parser.ChromeDataParser;
-import cn.reghao.bnt.spider.url.BodyDataType;
-import cn.reghao.bnt.spider.url.Site;
-import lombok.extern.slf4j.Slf4j;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * @author reghao
- * @date 2023-08-31 00:20:54
- */
-@Slf4j
-public class TmallSpiderTest {
-    public static Map<String, String> map = new HashMap<>();
-    static ChromeDataParser chromeDataParser = new ChromeDataParser();
-
-    static Set<ReqMatcher> reqMatchers() {
-        Set<ReqMatcher> set = new HashSet<>();
-        String pattern = "h5/mtop.taobao.rate.detaillist.get/6.0";
-        String parser = "TmallCommentDataParser";
-        map.put(pattern, parser);
-        set.add(new ReqMatcher(Site.bilibili, pattern, BodyDataType.json, chromeDataParser));
-
-        String pattern1 = "rate.taobao.com/feedRateList.htm";
-        String parser1 = "TaobaoCommentDataParser";
-        map.put(pattern1, parser1);
-        set.add(new ReqMatcher(Site.bilibili, pattern1, BodyDataType.json, chromeDataParser));
-        return set;
-    }
-
-    public static void main(String[] args) {
-        Set<ReqMatcher> set = reqMatchers();
-
-        String url = "https://www.tmall.com/";
-        AbstractChrome chrome = new ChromeBrowser(false, false);
-        chrome.getAndHandleDynamicPage(url, set);
-    }
-}

crawler/Dockerfile

@@ -1,7 +0,0 @@
-FROM adoptopenjdk/openjdk11:x86_64-alpine-jre-11.0.15_10
-
-WORKDIR /app
-RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo 'Asia/Shanghai' >/etc/timezone
-
-COPY target/spider-crawler-exec.jar /app/spider-crawler-exec.jar
-ENTRYPOINT ["java","-Djava.security.egd=file:/dev/./urandom","-jar","/app/spider-crawler-exec.jar"]

crawler/README.md

@@ -1 +0,0 @@
-# crawler

crawler/bin/shutdown.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-app='spider-crawler-exec.jar'
-pid=`jps | grep ${app} | awk '{print $1}'`
-kill -15 ${pid}

crawler/bin/start.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-app='spider-crawler-exec.jar'
-app_path=`pwd`/${app}
-nohup java -jar ${app_path} > console.log 2>&1 &

crawler/dependency-reduced-pom.xml

@@ -1,51 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-  <parent>
-    <artifactId>bnt</artifactId>
-    <groupId>cn.reghao.bnt</groupId>
-    <version>1.0.0</version>
-  </parent>
-  <modelVersion>4.0.0</modelVersion>
-  <artifactId>crawler</artifactId>
-  <version>1.0.0-SNAPSHOT</version>
-  <build>
-    <finalName>bnt-crawler</finalName>
-    <plugins>
-      <plugin>
-        <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.8.1</version>
-        <configuration>
-          <source>11</source>
-          <target>11</target>
-        </configuration>
-      </plugin>
-      <plugin>
-        <artifactId>maven-shade-plugin</artifactId>
-        <version>3.2.4</version>
-        <executions>
-          <execution>
-            <phase>package</phase>
-            <goals>
-              <goal>shade</goal>
-            </goals>
-            <configuration>
-              <transformers>
-                <transformer>
-                  <mainClass>cn.reghao.bnt.crawler.SpiderCrawler</mainClass>
-                </transformer>
-              </transformers>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-    </plugins>
-  </build>
-  <dependencies>
-    <dependency>
-      <groupId>org.projectlombok</groupId>
-      <artifactId>lombok</artifactId>
-      <version>1.18.0</version>
-      <scope>provided</scope>
-    </dependency>
-  </dependencies>
-</project>

crawler/pom.xml

@@ -1,58 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>bnt</artifactId>
-        <groupId>cn.reghao.bnt</groupId>
-        <version>1.0.0</version>
-    </parent>
-    <modelVersion>4.0.0</modelVersion>
-
-    <artifactId>crawler</artifactId>
-    <version>1.0.0-SNAPSHOT</version>
-
-    <dependencies>
-        <dependency>
-            <groupId>cn.reghao.bnt</groupId>
-            <artifactId>spider</artifactId>
-            <version>1.0.0-SNAPSHOT</version>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <finalName>bnt-crawler</finalName>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-compiler-plugin</artifactId>
-                <version>3.8.1</version>
-                <configuration>
-                    <source>11</source>
-                    <target>11</target>
-                </configuration>
-            </plugin>
-
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-shade-plugin</artifactId>
-                <version>3.2.4</version>
-                <executions>
-                    <execution>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>shade</goal>
-                        </goals>
-                        <configuration>
-                            <transformers>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-                                    <mainClass>cn.reghao.bnt.crawler.SpiderCrawler</mainClass>
-                                </transformer>
-                            </transformers>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-</project>

crawler/src/main/java/cn/reghao/bnt/crawler/Crawler.java

@@ -1,74 +0,0 @@
-package cn.reghao.bnt.crawler;
-
-import cn.reghao.bnt.crawler.task.SiteParserGetter;
-import cn.reghao.jutil.jdk.thread.ThreadPoolWrapper;
-import cn.reghao.bnt.spider.http.CrawlRequest;
-import cn.reghao.bnt.spider.http.DefaultCrawlRequest;
-import cn.reghao.bnt.spider.parser.SiteParser;
-import cn.reghao.bnt.spider.url.UrlScheduler;
-import cn.reghao.bnt.crawler.impl.HttpUrlScheduler;
-import cn.reghao.bnt.crawler.task.DataProducer;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-
-/**
- * @author reghao
- * @date 2022-02-25 16:55:41
- */
-public class Crawler {
-    private final CrawlRequest crawlRequest;
-    private final UrlScheduler urlScheduler;
-    private final ScheduledExecutorService crawlerScheduler;
-    private final Map<SiteParser, ScheduledFuture<?>> map;
-
-    public Crawler(String managerUrl, int maxThread) {
-        this.crawlRequest = new DefaultCrawlRequest();
-        this.urlScheduler = new HttpUrlScheduler(managerUrl);
-        this.crawlerScheduler = ThreadPoolWrapper.scheduledThreadPool("crawler", maxThread);
-        this.map = new ConcurrentHashMap<>();
-    }
-
-    public ScheduledExecutorService getCrawlerScheduler() {
-        return crawlerScheduler;
-    }
-
-    public CrawlRequest getCrawlRequest() {
-        return crawlRequest;
-    }
-
-    public UrlScheduler getUrlScheduler() {
-        return urlScheduler;
-    }
-
-    public void start() {
-        SiteParserGetter siteParserGetter = new SiteParserGetter(urlScheduler, this);
-        crawlerScheduler.scheduleAtFixedRate(siteParserGetter, 0, 1, TimeUnit.MINUTES);
-    }
-
-    public void start(SiteParser siteParser) {
-        if (map.get(siteParser) == null) {
-            DataProducer dataProducer = new DataProducer(this, siteParser);
-            ScheduledFuture<?> future = crawlerScheduler.scheduleAtFixedRate(dataProducer, 1, 3, TimeUnit.SECONDS);
-            map.put(siteParser, future);
-        }
-    }
-
-    public void stop() {
-        map.forEach(((siteParser, scheduledFuture) -> {
-            scheduledFuture.cancel(true);
-        }));
-        map.clear();
-    }
-
-    public void stop(SiteParser siteParser) {
-        ScheduledFuture<?> future = map.get(siteParser);
-        if (future != null) {
-            future.cancel(true);
-        }
-        map.remove(siteParser);
-    }
-}

crawler/src/main/java/cn/reghao/bnt/crawler/CrawlerContext.java

@@ -1,82 +0,0 @@
-package cn.reghao.bnt.crawler;
-
-import cn.reghao.jutil.jdk.thread.ThreadPoolWrapper;
-import cn.reghao.jutil.jdk.util.SingleInstance;
-
-import java.util.logging.Logger;
-
-/**
- * 数据爬取上下文
- *
- * @author reghao
- * @date 2019-11-29 17:34:01
- */
-public class CrawlerContext {
-    private static final Logger log = Logger.getLogger(CrawlerContext.class.getName());
-
-    private final Crawler crawler;
-
-    public CrawlerContext(Crawler crawler) {
-        this.crawler = crawler;
-    }
-
-    /**
-     * 开始数据爬取任务
-     *
-     * @param
-     * @return
-     * @date 2020-03-16 下午8:35
-     */
-    public void start() {
-        crawler.start();
-        stopGracefully();
-        keepMainRunning();
-    }
-
-    /**
-     * 主线程保持运行状态
-     *
-     * @param
-     * @return
-     * @date 2021-08-09 上午12:12
-     */
-    private void keepMainRunning() {
-        SingleInstance.onlyOne(60002);
-    }
-
-    /**
-     * 优雅关闭爬虫（进程级别的关闭）
-     *
-     * @date 2019-08-16 下午3:45
-     */
-    private void stopGracefully() {
-        Runtime.getRuntime().addShutdownHook(new Thread(new ShutdownHook(), "main-shutdown-hook"));
-    }
-
-    /**
-     * 1.可以处理 kill -2 或 kill -15
-     * 2.无法处理 kill -9 和机器断电的情况
-     *
-     * @date 2019-08-16 下午4:07
-     */
-    class ShutdownHook implements Runnable {
-        @Override
-        public void run() {
-            stop();
-        }
-    }
-
-    /**
-     * 1.停止从 UrlScheduler 中获取数据
-     * 2.停止向 DataProducer 提交数据
-     * 3.线程池停止接受新任务，并等待当前执行的任务完成
-     *
-     * @param
-     * @return
-     * @date 2021-04-02 下午11:26
-     */
-    private void stop() {
-        ThreadPoolWrapper.shutdownScheduler(crawler.getCrawlerScheduler());
-        log.info("resource cleared and terminate SpiderCrawler...");
-    }
-}

crawler/src/main/java/cn/reghao/bnt/crawler/SpiderCrawler.java

@@ -1,21 +0,0 @@
-package cn.reghao.bnt.crawler;
-
-import java.util.logging.Logger;
-
-/**
- * 数据爬取应用
- *
- * @author reghao
- * @date 2020-04-15 09:46:09
- */
-public class SpiderCrawler {
-    private static final Logger log = Logger.getLogger(SpiderCrawler.class.getName());
-
-    public static void main(String[] args) {
-        String managerUrl = "http://spider.reghao.cn";
-        final Crawler crawler = new Crawler(managerUrl, 10);
-        CrawlerContext crawlerContext = new CrawlerContext(crawler);
-        log.info("starting SpiderCralwer...");
-        //crawlerContext.start();
-    }
-}

crawler/src/main/java/cn/reghao/bnt/crawler/impl/HttpUrlScheduler.java

@@ -1,115 +0,0 @@
-package cn.reghao.bnt.crawler.impl;
-
-import cn.reghao.jutil.jdk.serializer.JsonConverter;
-import cn.reghao.bnt.spider.url.UrlScheduler;
-import cn.reghao.bnt.spider.event.EvtCrawl;
-import cn.reghao.bnt.spider.url.RawData;
-import cn.reghao.bnt.spider.url.CrawlUrl;
-import cn.reghao.bnt.spider.parser.SiteParser;
-import com.google.gson.JsonObject;
-
-import java.net.URI;
-import java.net.http.HttpClient;
-import java.net.http.HttpRequest;
-import java.net.http.HttpResponse;
-import java.time.Duration;
-import java.util.*;
-
-/**
- * @author reghao
- * @date 2022-05-25 23:44:25
- */
-public class HttpUrlScheduler implements UrlScheduler {
-    private final String managerUrl;
-    private final HttpClient client = HttpClient.newBuilder()
-            .version(HttpClient.Version.HTTP_1_1)
-            .build();
-    
-    public HttpUrlScheduler(String managerUrl) {
-        this.managerUrl = managerUrl;
-    }
-
-    @Override
-    public List<SiteParser> getSiteParsers() {
-        try {
-            HttpRequest request = HttpRequest.newBuilder()
-                    .uri(URI.create(managerUrl + "/api/crawler/siteparsers"))
-                    .timeout(Duration.ofSeconds(30))
-                    .GET()
-                    .build();
-            HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
-            int statusCode = response.statusCode();
-            if (statusCode == 200) {
-                String json = response.body();
-                JsonObject jsonObject = JsonConverter.jsonToJsonElement(json).getAsJsonObject();
-                int code = jsonObject.get("code").getAsInt();
-                if (code == 0) {
-                    String json1 = jsonObject.get("data").toString();
-                    List<SiteParser> list = JsonConverter.jsonToObjects(json1, SiteParser.class);
-                    return list;
-                }
-            }
-        } catch (Exception ignore) {
-        }
-        return Collections.emptyList();
-    }
-
-    @Override
-    public Set<CrawlUrl> getUrls(String site, String parser) {
-        String url = String.format(managerUrl + "/api/crawler/urls?site=%s&parser=%s", site, parser);
-        HttpRequest request = HttpRequest.newBuilder()
-                .uri(URI.create(url))
-                .timeout(Duration.ofSeconds(30 ))
-                .GET()
-                .build();
-        try {
-            HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
-            int statusCode = response.statusCode();
-            if (statusCode == 200) {
-                String json = response.body();
-                JsonObject jsonObject = JsonConverter.jsonToJsonElement(json).getAsJsonObject();
-                int code = jsonObject.get("code").getAsInt();
-                if (code == 0) {
-                    String json1 = jsonObject.get("data").toString();
-                    List<CrawlUrl> list = JsonConverter.jsonToObjects(json1, CrawlUrl.class);
-                    return new HashSet<>(list);
-                }
-            }
-        } catch (Exception ignore) {
-        }
-
-        return Collections.emptySet();
-    }
-
-    @Override
-    public void putData(RawData rawData) {
-        String jsonPayload = JsonConverter.objectToJson(rawData);
-        HttpRequest request = HttpRequest.newBuilder()
-                .uri(URI.create(managerUrl + "/api/crawler/rawdata"))
-                .timeout(Duration.ofSeconds(30))
-                .header("Content-Type", "application/json")
-                .POST(HttpRequest.BodyPublishers.ofString(jsonPayload))
-                .build();
-        try {
-            HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
-            int statusCode = response.statusCode();
-        } catch (Exception ignore) {
-        }
-    }
-
-    @Override
-    public void crawlEvent(EvtCrawl evtCrawl) {
-        String jsonPayload = JsonConverter.objectToJson(evtCrawl);
-        HttpRequest request = HttpRequest.newBuilder()
-                .uri(URI.create(managerUrl + "/api/crawler/event/crawl"))
-                .timeout(Duration.ofSeconds(30))
-                .header("Content-Type", "application/json")
-                .POST(HttpRequest.BodyPublishers.ofString(jsonPayload))
-                .build();
-        try {
-            HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
-            int statusCode = response.statusCode();
-        } catch (Exception ignore) {
-        }
-    }
-}

crawler/src/main/java/cn/reghao/bnt/crawler/task/DataProducer.java

@@ -1,104 +0,0 @@
-package cn.reghao.bnt.crawler.task;
-
-import cn.reghao.bnt.crawler.Crawler;
-import cn.reghao.bnt.spider.url.RawData;
-import cn.reghao.bnt.spider.event.EvtCrawl;
-import cn.reghao.bnt.spider.http.CrawlRequest;
-import cn.reghao.bnt.spider.http.CrawlResponse;
-import cn.reghao.bnt.spider.url.CrawlUrl;
-import cn.reghao.bnt.spider.parser.SiteParser;
-import cn.reghao.bnt.spider.url.UrlScheduler;
-
-import java.util.LinkedList;
-import java.util.Queue;
-import java.util.Set;
-import java.util.logging.Logger;
-
-/**
- * @author reghao
- * @date 2022-05-18 11:38:16
- */
-public class DataProducer implements Runnable {
-    private static final Logger log = Logger.getLogger(DataProducer.class.getName());
-
-    private final Crawler crawler;
-    private final SiteParser siteParser;
-    private final CrawlRequest crawlRequest;
-    private final UrlScheduler urlScheduler;
-    private int failedCount;
-    private boolean antiAntiCrawl = false;
-    private final Queue<CrawlUrl> cachedCrawlUrls = new LinkedList<>();
-
-    public DataProducer(Crawler crawler, SiteParser siteParser) {
-        this.crawler = crawler;
-        this.siteParser = siteParser;
-        this.urlScheduler = crawler.getUrlScheduler();
-        this.crawlRequest = crawler.getCrawlRequest();
-    }
-
-    @Override
-    public void run() {
-        String site = siteParser.getSite();
-        String parser = siteParser.getParser();
-        try {
-            if (antiAntiCrawl()) {
-                return;
-            }
-
-            CrawlUrl crawlUrl = cachedCrawlUrls.poll();
-            if (crawlUrl == null) {
-                Set<CrawlUrl> set = urlScheduler.getUrls(site, parser);
-                if (!set.isEmpty()) {
-                    cachedCrawlUrls.addAll(set);
-                } else {
-                    crawler.stop(siteParser);
-                }
-                return;
-            }
-            crawl(crawlUrl);
-        } catch (Exception ignore) {
-        }
-    }
-
-    /**
-     * 反反爬虫策略
-     *
-     * @param
-     * @return
-     * @date 2021-08-13 下午2:47
-     */
-    private boolean antiAntiCrawl() {
-        if (failedCount > 10 && !antiAntiCrawl) {
-            failedCount = failedCount * 10;
-            antiAntiCrawl = true;
-        }
-
-        if (failedCount < 0) {
-            antiAntiCrawl = false;
-        }
-
-        if (antiAntiCrawl) {
-            log.severe(String.format("执行 %s 的反反爬虫策略, 还需跳过 %s 次请求", siteParser.getSite(), failedCount));
-            failedCount--;
-        }
-
-        return antiAntiCrawl;
-    }
-
-    private void crawl(CrawlUrl crawlUrl) {
-        EvtCrawl evtCrawl = new EvtCrawl(siteParser.getSite(), siteParser.getParser(), crawlUrl.getUrl());
-        CrawlResponse response = crawlRequest.get(crawlUrl);
-        int statusCode = response.getStatusCode();
-        if (statusCode == 200) {
-            urlScheduler.putData(new RawData(crawlUrl, response.getBody()));
-            urlScheduler.crawlEvent(evtCrawl);
-        } else if (statusCode == 404 || statusCode == crawlUrl.getNotFoundCode()) {
-            urlScheduler.putData(new RawData(crawlUrl, "404"));
-            urlScheduler.crawlEvent(evtCrawl);
-        } else {
-            failedCount++;
-            evtCrawl.setMsg(response.getBody());
-            urlScheduler.crawlEvent(evtCrawl);
-        }
-    }
-}

crawler/src/main/java/cn/reghao/bnt/crawler/task/SiteParserGetter.java

@@ -1,23 +0,0 @@
-package cn.reghao.bnt.crawler.task;
-
-import cn.reghao.bnt.spider.url.UrlScheduler;
-import cn.reghao.bnt.crawler.Crawler;
-
-/**
- * @author reghao
- * @date 2022-05-18 11:43:36
- */
-public class SiteParserGetter implements Runnable {
-    private final UrlScheduler urlScheduler;
-    private final Crawler crawler;
-
-    public SiteParserGetter(UrlScheduler urlScheduler, Crawler crawler) {
-        this.urlScheduler = urlScheduler;
-        this.crawler = crawler;
-    }
-
-    @Override
-    public void run() {
-        urlScheduler.getSiteParsers().forEach(crawler::start);
-    }
-}

pom.xml

@@ -13,10 +13,6 @@
         <module>agent</module>
         <module>logstash</module>
         <module>deployer</module>
-        <module>spider</module>
-        <module>browser</module>
-        <module>crawler</module>
-        <module>admin-api</module>
     </modules>
     <packaging>pom</packaging>
 

spider/pom.xml

@@ -1,14 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>bnt</artifactId>
-        <groupId>cn.reghao.bnt</groupId>
-        <version>1.0.0</version>
-    </parent>
-    <modelVersion>4.0.0</modelVersion>
-
-    <artifactId>spider</artifactId>
-    <version>1.0.0-SNAPSHOT</version>
-</project>

spider/src/main/java/cn/reghao/bnt/spider/event/EvtCrawl.java

@@ -1,56 +0,0 @@
-package cn.reghao.bnt.spider.event;
-
-import cn.reghao.jutil.jdk.event.message.Event;
-
-/**
- * @author reghao
- * @date 2022-02-25 18:52:57
- */
-public class EvtCrawl extends Event {
-    private String site;
-    private String parser;
-    private String url;
-    private String msg;
-
-    public EvtCrawl() {
-    }
-
-    public EvtCrawl(String site, String parser, String url) {
-        this.site = site;
-        this.parser = parser;
-        this.url = url;
-        this.msg = "ok";
-    }
-
-    public void setSite(String site) {
-        this.site = site;
-    }
-
-    public String getSite() {
-        return site;
-    }
-
-    public void setParser(String parser) {
-        this.parser = parser;
-    }
-
-    public String getParser() {
-        return parser;
-    }
-
-    public void setUrl(String url) {
-        this.url = url;
-    }
-
-    public String getUrl() {
-        return url;
-    }
-
-    public void setMsg(String msg) {
-        this.msg = msg;
-    }
-
-    public String getMsg() {
-        return msg;
-    }
-}

spider/src/main/java/cn/reghao/bnt/spider/event/EvtNoUrl.java

@@ -1,12 +0,0 @@
-package cn.reghao.bnt.spider.event;
-
-import cn.reghao.jutil.jdk.event.message.Event;
-
-/**
- * @author reghao
- * @date 2021-12-17 17:19:50
- */
-public class EvtNoUrl extends Event {
-    private String site;
-    private String parser;
-}

spider/src/main/java/cn/reghao/bnt/spider/http/CrawlRequest.java

@@ -1,18 +0,0 @@
-package cn.reghao.bnt.spider.http;
-
-import cn.reghao.bnt.spider.url.CrawlUrl;
-
-import java.io.File;
-
-/**
- * 爬虫请求
- *
- * @author reghao
- * @date 2019-08-01 16:27:55
- */
-public interface CrawlRequest {
-    int head(String url);
-    CrawlResponse get(String url);
-    CrawlResponse get(CrawlUrl crawlUrl);
-    boolean download(CrawlUrl crawlUrl, File file);
-}

spider/src/main/java/cn/reghao/bnt/spider/http/CrawlResponse.java

@@ -1,25 +0,0 @@
-package cn.reghao.bnt.spider.http;
-
-/**
- * 爬虫响应
- *
- * @author reghao
- * @date 2019-08-01 16:27:55
- */
-public class CrawlResponse {
-    private int statusCode;
-    private String body;
-
-    public CrawlResponse(int statusCode, String body) {
-        this.statusCode = statusCode;
-        this.body = body;
-    }
-
-    public int getStatusCode() {
-        return statusCode;
-    }
-
-    public String getBody() {
-        return body;
-    }
-}

spider/src/main/java/cn/reghao/bnt/spider/http/DefaultCrawlRequest.java

@@ -1,109 +0,0 @@
-package cn.reghao.bnt.spider.http;
-
-import cn.reghao.jutil.tool.http.BaseWebRequest;
-import cn.reghao.jutil.jdk.http.util.UserAgents;
-import cn.reghao.bnt.spider.url.CrawlUrl;
-import org.apache.http.*;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpHead;
-import org.apache.http.client.methods.HttpRequestBase;
-import org.apache.http.cookie.Cookie;
-import org.apache.http.protocol.HttpContext;
-import org.apache.http.util.EntityUtils;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.text.MessageFormat;
-import java.util.List;
-import java.util.logging.Logger;
-
-/**
- * @author reghao
- * @date 2019-11-29 10:03:18
- */
-public class DefaultCrawlRequest extends BaseWebRequest implements CrawlRequest {
-    private static final Logger log = Logger.getLogger(DefaultCrawlRequest.class.getName());
-
-    public DefaultCrawlRequest() {
-    }
-
-    @Override
-    public int head(String url) {
-        HttpHead head = new HttpHead(url);
-        try (CloseableHttpResponse response = client.execute(head)) {
-            return response.getStatusLine().getStatusCode();
-        } catch (IOException e) {
-            log.info(MessageFormat.format("{0} head 请求失败 -> {1}", url, e.getMessage()));
-        }
-
-        // 资源无法访问
-        return 600;
-    }
-
-    @Override
-    public CrawlResponse get(String url) {
-        HttpGet get = new HttpGet(url);
-        get.setHeader("User-Agent", UserAgents.getDesktopAgent());
-        return execRequest(get, context);
-    }
-
-    public CrawlResponse get(String url, List<Cookie> cookies) {
-        HttpGet get = new HttpGet(url);
-        get.setHeader("User-Agent", UserAgents.getMobileAgent());
-        return execRequest(get, context);
-    }
-
-    @Override
-    public CrawlResponse get(CrawlUrl crawlUrl) {
-        HttpGet get = new HttpGet(crawlUrl.getUrl());
-        String referer = crawlUrl.getReferer();
-        if (referer != null) {
-            get.setHeader("Referer", referer);
-        }
-        get.setHeader("User-Agent", UserAgents.getDesktopAgent());
-        return execRequest(get, context);
-    }
-
-    private CrawlResponse execRequest(HttpRequestBase request, HttpContext context) {
-        try (CloseableHttpResponse response = client.execute(request, context)) {
-            StatusLine statusLine = response.getStatusLine();
-            int statusCode = statusLine.getStatusCode();
-            String body = EntityUtils.toString(response.getEntity(), Charset.forName(bodyCharset));
-            return new CrawlResponse(statusCode, body);
-        } catch (Exception e) {
-            // TODO 是否应该放在 finally 块中？
-            return new CrawlResponse(600, e.getMessage());
-        }
-    }
-
-    @Override
-    public boolean download(CrawlUrl crawlUrl, File file) {
-        String url = crawlUrl.getUrl();
-        String referer = crawlUrl.getReferer();
-        HttpGet get = new HttpGet(url);
-        if (referer != null) {
-            get.setHeader("Referer", referer);
-        }
-        get.setHeader("User-Agent", UserAgents.getDesktopAgent());
-        long start = System.currentTimeMillis();
-        try (CloseableHttpResponse response = client.execute(get)) {
-            int statusCode = response.getStatusLine().getStatusCode();
-            if (statusCode == 200) {
-                HttpEntity httpEntity = response.getEntity();
-                String contentType = httpEntity.getContentType().getValue();
-                //String filename = UrlFormatter.getFilename(url);
-
-                FileOutputStream fout = new FileOutputStream(file);
-                // 持续写到本地文件，直到服务器没有数据
-                httpEntity.writeTo(fout);
-                return true;
-            }
-        } catch (IOException e) {
-            log.info(MessageFormat.format("{0} 下载失败 -> {1}", url, e.getMessage()));
-        }
-        return false;
-    }
-}

spider/src/main/java/cn/reghao/bnt/spider/http/JdkCrawlRequest.java

@@ -1,131 +0,0 @@
-package cn.reghao.bnt.spider.http;
-
-import cn.reghao.bnt.spider.url.CrawlUrl;
-import cn.reghao.jutil.jdk.http.util.UserAgents;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.net.http.HttpClient;
-import java.net.http.HttpRequest;
-import java.net.http.HttpResponse;
-import java.text.MessageFormat;
-import java.time.Duration;
-import java.util.logging.Logger;
-
-/**
- * 爬虫请求
- *
- * @author reghao
- * @date 2022-02-28 15:27:55
- */
-public class JdkCrawlRequest implements CrawlRequest {
-    private static final Logger log = Logger.getLogger(JdkCrawlRequest.class.getName());
-
-    private final HttpClient client = HttpClient.newBuilder()
-            .version(HttpClient.Version.HTTP_1_1)
-            .build();
-
-    @Override
-    public int head(String url) {
-        HttpRequest.Builder builder = HttpRequest.newBuilder()
-                .uri(URI.create(url))
-                .timeout(Duration.ofSeconds(5))
-                .GET();
-        builder.setHeader("User-Agent", UserAgents.getDesktopAgent());
-
-        try {
-            HttpResponse<String> response = client.send(builder.build(), HttpResponse.BodyHandlers.ofString());
-            return response.statusCode();
-        } catch (Exception e) {
-            log.info(MessageFormat.format("{0} 请求失败 -> {1}", url, e.getMessage()));
-            return 600;
-        }
-    }
-
-    @Override
-    public CrawlResponse get(String url) {
-        HttpRequest.Builder builder = HttpRequest.newBuilder()
-                .uri(URI.create(url))
-                .timeout(Duration.ofSeconds(5))
-                .GET();
-        builder.setHeader("User-Agent", UserAgents.getDesktopAgent());
-
-        try {
-            HttpResponse<String> response = client.send(builder.build(), HttpResponse.BodyHandlers.ofString());
-            int statusCode = response.statusCode();
-            // TODO 处理返回的内容编码, 默认编码是 utf-8
-            String body = response.body();
-            return new CrawlResponse(statusCode, body);
-        } catch (Exception e) {
-            log.info(MessageFormat.format("{0} 请求失败 -> {1}", url, e.getMessage()));
-            return new CrawlResponse(600, e.getMessage());
-        }
-    }
-
-    @Override
-    public CrawlResponse get(CrawlUrl crawlUrl) {
-        String url = crawlUrl.getUrl();
-        HttpRequest.Builder builder = HttpRequest.newBuilder()
-                .uri(URI.create(url))
-                .timeout(Duration.ofSeconds(5))
-                .GET();
-        builder.setHeader("User-Agent", UserAgents.getDesktopAgent());
-        String referer = crawlUrl.getReferer();
-        if (referer != null) {
-            builder.setHeader("Referer", referer);
-        }
-
-        try {
-            HttpResponse<String> response = client.send(builder.build(), HttpResponse.BodyHandlers.ofString());
-            int statusCode = response.statusCode();
-            String body = response.body();
-            return new CrawlResponse(statusCode, body);
-        } catch (Exception e) {
-            log.info(MessageFormat.format("{0} 请求失败 -> {1}", url, e.getMessage()));
-            return new CrawlResponse(600, e.getMessage());
-        }
-    }
-
-    @Override
-    public boolean download(CrawlUrl crawlUrl, File file) {
-        String url = crawlUrl.getUrl();
-        HttpRequest.Builder builder = HttpRequest.newBuilder()
-                .uri(URI.create(url))
-                .timeout(Duration.ofSeconds(30))
-                .GET();
-        builder.setHeader("User-Agent", UserAgents.getDesktopAgent());
-        String referer = crawlUrl.getReferer();
-        if (referer != null) {
-            builder.setHeader("Referer", referer);
-        }
-
-        try {
-            HttpResponse<InputStream> in = client.send(builder.build(), HttpResponse.BodyHandlers.ofInputStream());
-            saveFile(in.body(), file);
-            return true;
-        } catch (Exception e) {
-            log.info(MessageFormat.format("{0} 下载失败 -> {1}", url, e.getMessage()));
-        }
-        return false;
-    }
-
-    private void saveFile(InputStream in, File file) throws IOException {
-        File parentDir = file.getParentFile();
-        if (!parentDir.exists()) {
-            //FileUtils.forceMkdir(parentDir);
-        }
-
-        FileOutputStream fos = new FileOutputStream(file);
-        // 1MiB
-        int len = 1024*1024;
-        byte[] buf = new byte[len];
-        int readLen;
-        while ((readLen = in.read(buf, 0, len)) != -1) {
-            fos.write(buf, 0, readLen);
-        }
-        fos.close();
-    }
-}

spider/src/main/java/cn/reghao/bnt/spider/parser/DataParser.java

@@ -1,21 +0,0 @@
-package cn.reghao.bnt.spider.parser;
-
-import java.util.Map;
-
-/**
- * 数据解析器，解析 crawler 爬取的数据
- *
- * @author reghao
- * @date 2019-11-29 17:55:17
- */
-public interface DataParser {
-    /**
-     * parser 名字(即 DataParser 实现类名)应该全局唯一
-     *
-     * @date 2021-02-03 下午1:55
-     */
-    default String parserName() {
-        return this.getClass().getSimpleName();
-    }
-    Map<String, Object> parse(String url, String body) throws Exception;
-}

spider/src/main/java/cn/reghao/bnt/spider/parser/SiteParser.java

@@ -1,54 +0,0 @@
-package cn.reghao.bnt.spider.parser;
-
-import java.io.Serializable;
-
-/**
- * @author reghao
- * @date 2022-05-18 11:55:19
- */
-public class SiteParser implements Serializable {
-    private static final long serialVersionUID = 1L;
-
-    private final String site;
-    private final String parser;
-
-    public SiteParser(String site, String parser) {
-        this.site = site;
-        this.parser = parser;
-    }
-
-    public String getSite() {
-        return site;
-    }
-
-    public String getParser() {
-        return parser;
-    }
-
-    @Override
-    public String toString() {
-        return String.format("%s.%s", site, parser);
-    }
-
-    @Override
-    public int hashCode() {
-        int result = 17;
-        result = result * 31 + site.hashCode();
-        result = result * 31 + parser.hashCode();
-        return result;
-    }
-
-    @Override
-    public boolean equals(Object other) {
-        if (other == this) {
-            return true;
-        }
-
-        if (other instanceof SiteParser) {
-            SiteParser o = (SiteParser) other;
-            return o.site.equals(site) && o.parser.equals(parser);
-        } else {
-            return false;
-        }
-    }
-}

spider/src/main/java/cn/reghao/bnt/spider/url/BodyDataType.java

@@ -1,11 +0,0 @@
-package cn.reghao.bnt.spider.url;
-
-/**
- * HTTP 响应数据的格式
- *
- * @author reghao
- * @date 2020-04-10 20:56:21
- */
-public enum BodyDataType {
-    json, html, text, m3u8
-}

spider/src/main/java/cn/reghao/bnt/spider/url/CrawlUrl.java

@@ -1,148 +0,0 @@
-package cn.reghao.bnt.spider.url;
-
-import java.io.Serializable;
-
-/**
- * @author reghao
- * @date 2020-03-15 13:53:38
- */
-public class CrawlUrl implements Serializable {
-    private static final long serialVersionUID = 1L;
-
-    private String site;
-    // DataParser 实现类的名字(确保全局唯一)
-    private String parser;
-    private String url;
-    // 若 url 是一个接口，那 referer 表示发送接口请求的页面 url
-    private String referer;
-    // 响应 body 的数据类型，BodyDataType 枚举中的值
-    private String dataType;
-    // 资源不存在时返回的 http status code
-    private int notFoundCode;
-    // 由于反爬虫返回的 http status code
-    private int antiCrawlCode;
-    private String title;
-
-    public CrawlUrl() {
-    }
-
-    public CrawlUrl(String site, String parser, String url) {
-        this.site = site;
-        this.parser = parser;
-        this.url = url;
-        this.referer = null;
-        this.dataType = null;
-        this.notFoundCode = 404;
-        this.antiCrawlCode = 403;
-        this.title = null;
-    }
-
-    public CrawlUrl(String site, String parser, String url, String referer, String dataType,
-                    int notFoundCode, int antiCrawlCode, String title) {
-        this.site = site;
-        this.parser = parser;
-        this.url = url;
-        this.referer = referer;
-        this.dataType = dataType;
-        this.notFoundCode = notFoundCode;
-        this.antiCrawlCode = antiCrawlCode;
-        this.title = title;
-    }
-
-    @Deprecated
-    public CrawlUrl(String url, String referer) {
-        this.url = url;
-        this.referer = referer;
-    }
-
-    @Deprecated
-    public void setSiteAndParser(String site, String parser) {
-        this.site = site;
-        this.parser = parser;
-    }
-
-    public void setSite(String site) {
-        this.site = site;
-    }
-
-    public String getSite() {
-        return site;
-    }
-
-    public void setParser(String parser) {
-        this.parser = parser;
-    }
-
-    public String getParser() {
-        return parser;
-    }
-
-    public void setUrl(String url) {
-        this.url = url;
-    }
-
-    public String getUrl() {
-        return url;
-    }
-
-    public void setReferer(String referer) {
-        this.referer = referer;
-    }
-
-    public String getReferer() {
-        return referer;
-    }
-
-    public void setDataType(String dataType) {
-        this.dataType = dataType;
-    }
-
-    public String getDataType() {
-        return dataType;
-    }
-
-    public void setNotFoundCode(int notFoundCode) {
-        this.notFoundCode = notFoundCode;
-    }
-
-    public int getNotFoundCode() {
-        return notFoundCode;
-    }
-
-    public void setAntiCrawlCode(int antiCrawlCode) {
-        this.antiCrawlCode = antiCrawlCode;
-    }
-
-    public int getAntiCrawlCode() {
-        return antiCrawlCode;
-    }
-
-    public void setTitle(String title) {
-        this.title = title;
-    }
-
-    public String getTitle() {
-        return title;
-    }
-
-    @Override
-    public int hashCode() {
-        int result = 17;
-        result = result * 31 + url.hashCode();
-        return result;
-    }
-
-    @Override
-    public boolean equals(Object other) {
-        if (other == this) {
-            return true;
-        }
-
-        if (other instanceof CrawlUrl) {
-            CrawlUrl o = (CrawlUrl) other;
-            return o.url.equals(url);
-        } else {
-            return false;
-        }
-    }
-}

spider/src/main/java/cn/reghao/bnt/spider/url/RawData.java

@@ -1,73 +0,0 @@
-package cn.reghao.bnt.spider.url;
-
-import java.io.Serializable;
-
-/**
- * 在 DataBus 中传递的未解析的原始数据
- *
- * @author reghao
- * @date 2019-12-09 09:05:48
- */
-public class RawData implements Serializable {
-    private static final long serialVersionUID = 1L;
-
-    private CrawlUrl crawlUrl;
-    private String data;
-    private long crawledTime;
-
-    public RawData() {
-    }
-
-    public RawData(CrawlUrl crawlUrl, String data) {
-        this.crawlUrl = crawlUrl;
-        this.data = data;
-        this.crawledTime = System.currentTimeMillis();
-    }
-
-    public void setCrawlUrl(CrawlUrl crawlUrl) {
-        this.crawlUrl = crawlUrl;
-    }
-
-    public CrawlUrl getCrawlUrl() {
-        return crawlUrl;
-    }
-
-    public void setData(String data) {
-        this.data = data;
-    }
-
-    public String getData() {
-        return data;
-    }
-
-    public void setCrawledTime(long crawledTime) {
-        this.crawledTime = crawledTime;
-    }
-
-    public long getCrawledTime() {
-        return crawledTime;
-    }
-
-    @Override
-    public int hashCode() {
-        int result = 17;
-        result = result * 31 + crawlUrl.hashCode();
-        result = result * 31 + data.hashCode();
-        return result;
-    }
-
-    @Override
-    public boolean equals(Object other) {
-        if (other == this) {
-            return true;
-        }
-
-        if (other instanceof RawData) {
-            RawData o = (RawData) other;
-            return o.crawlUrl.equals(crawlUrl)
-                    && o.data.equals(data);
-        } else {
-            return false;
-        }
-    }
-}

spider/src/main/java/cn/reghao/bnt/spider/url/Site.java

@@ -1,9 +0,0 @@
-package cn.reghao.bnt.spider.url;
-
-/**
- * @author reghao
- * @date 2020-03-15 19:07:39
- */
-public enum Site {
-    zhihu, weibo, bilibili, taobao
-}

spider/src/main/java/cn/reghao/bnt/spider/url/UrlScheduler.java

@@ -1,23 +0,0 @@
-package cn.reghao.bnt.spider.url;
-
-import cn.reghao.bnt.spider.event.EvtCrawl;
-import cn.reghao.bnt.spider.event.EvtNoUrl;
-import cn.reghao.bnt.spider.parser.SiteParser;
-
-import java.util.List;
-import java.util.Set;
-
-/**
- * URL 调度器
- *
- * @author reghao
- * @date 2019-08-06 11:24:24
- */
-public interface UrlScheduler {
-    List<SiteParser> getSiteParsers();
-    Set<CrawlUrl> getUrls(String site, String parser);
-    void putData(RawData rawData);
-    void crawlEvent(EvtCrawl evtCrawl);
-    default void noUrlEvent(EvtNoUrl evtNoUrl) {
-    }
-}

web/pom.xml

@@ -163,104 +163,6 @@
             <artifactId>commons-io</artifactId>
             <version>2.6</version>
         </dependency>
-
-        <!-- parser 依赖-->
-        <dependency>
-            <groupId>cn.reghao.jutil</groupId>
-            <artifactId>media</artifactId>
-            <version>1.0.0-SNAPSHOT</version>
-        </dependency>
-        <dependency>
-            <groupId>cn.reghao.bnt</groupId>
-            <artifactId>spider</artifactId>
-            <version>1.0.0-SNAPSHOT</version>
-        </dependency>
-        <dependency>
-            <groupId>cn.reghao.bnt</groupId>
-            <artifactId>browser</artifactId>
-            <version>1.0.0-SNAPSHOT</version>
-        </dependency>
-        <dependency>
-            <groupId>cn.reghao.oss</groupId>
-            <artifactId>oss-sdk</artifactId>
-            <version>1.0.0-SNAPSHOT</version>
-        </dependency>
-        <dependency>
-            <groupId>cn.reghao.tnb.account</groupId>
-            <artifactId>account-api</artifactId>
-            <version>1.0.0-SNAPSHOT</version>
-        </dependency>
-        <dependency>
-            <groupId>cn.reghao.tnb.content</groupId>
-            <artifactId>content-api</artifactId>
-            <version>1.0.0-SNAPSHOT</version>
-        </dependency>
-        <dependency>
-            <groupId>cn.reghao.tnb.data</groupId>
-            <artifactId>data-api</artifactId>
-            <version>1.0.0-SNAPSHOT</version>
-        </dependency>
-        <dependency>
-            <groupId>cn.reghao.tnb.user</groupId>
-            <artifactId>user-api</artifactId>
-            <version>1.0.0-SNAPSHOT</version>
-        </dependency>
-        <dependency>
-            <groupId>cn.reghao.tnb.mall</groupId>
-            <artifactId>mall-api</artifactId>
-            <version>1.0.0-SNAPSHOT</version>
-        </dependency>
-
-        <dependency>
-            <groupId>com.drewnoakes</groupId>
-            <artifactId>metadata-extractor</artifactId>
-            <version>2.18.0</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-imaging</artifactId>
-            <version>1.0-alpha3</version>
-        </dependency>
-
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-data-mongodb</artifactId>
-            <version>2.3.9.RELEASE</version>
-        </dependency>
-
-        <dependency>
-            <groupId>org.jsoup</groupId>
-            <artifactId>jsoup</artifactId>
-            <version>1.12.1</version>
-        </dependency>
-        <dependency>
-            <groupId>org.dom4j</groupId>
-            <artifactId>dom4j</artifactId>
-            <version>2.1.1</version>
-        </dependency>
-
-        <dependency>
-            <groupId>com.google.protobuf</groupId>
-            <artifactId>protobuf-java</artifactId>
-            <version>3.20.0-rc-1</version>
-        </dependency>
-
-        <!-- admin 依赖 -->
-        <dependency>
-            <groupId>cn.reghao.bnt</groupId>
-            <artifactId>admin-api</artifactId>
-            <version>1.0.0-SNAPSHOT</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.dubbo</groupId>
-            <artifactId>dubbo-spring-boot-starter</artifactId>
-            <version>2.7.8</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.curator</groupId>
-            <artifactId>curator-recipes</artifactId>
-            <version>2.12.0</version>
-        </dependency>
     </dependencies>
 
     <profiles>

web/src/main/java/cn/reghao/bnt/web/admin/controller/AccountController.java

@@ -1,57 +0,0 @@
-package cn.reghao.bnt.web.admin.controller;
-
-import cn.reghao.jutil.jdk.db.PageList;
-import cn.reghao.jutil.jdk.result.WebResult;
-import cn.reghao.bnt.web.admin.service.AdminAccountService;
-import cn.reghao.tnb.user.api.dto.UserInfo;
-import cn.reghao.tnb.user.api.dto.UserSearch;
-import cn.reghao.tnb.user.api.iface.AdminUserService;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import org.apache.dubbo.config.annotation.DubboReference;
-import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.*;
-
-/**
- * @author reghao
- * @date 2024-02-15 09:15:40
- */
-@Api(tags = "用户管理接口")
-@RestController
-@RequestMapping("/api/account")
-public class AccountController {
-    @DubboReference(check = false, retries = 0)
-    private AdminUserService adminUserService;
-
-    private AdminAccountService adminAccountService;
-
-    public AccountController(AdminAccountService adminAccountService) {
-        this.adminAccountService = adminAccountService;
-    }
-
-    @PostMapping("/create")
-    public String updateUserProfile() {
-        return WebResult.success();
-    }
-
-    @ApiOperation("获取用户列表")
-    @GetMapping(value = "/list", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String getUserList(UserSearch userSearch) {
-        PageList<UserInfo> pageList = adminUserService.getUserList(userSearch);
-        return WebResult.success(pageList);
-    }
-
-    @ApiOperation("同意用户充值")
-    @PostMapping(value = "/charge/approve/{chargeId}", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String approveCharge(@PathVariable("chargeId") Long chargeId) {
-        adminAccountService.approve(chargeId);
-        return WebResult.success();
-    }
-
-    @ApiOperation("拒绝用户充值")
-    @PostMapping(value = "/charge/decline/{chargeId}", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String declineCharge(@PathVariable("chargeId") Long chargeId) {
-        adminAccountService.decline(chargeId);
-        return WebResult.success();
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/controller/ContentController.java

@@ -1,33 +0,0 @@
-package cn.reghao.bnt.web.admin.controller;
-
-import cn.reghao.jutil.jdk.db.PageList;
-import cn.reghao.jutil.jdk.result.WebResult;
-import cn.reghao.tnb.content.api.dto.UserVideoPost;
-import cn.reghao.tnb.content.api.dto.VideoSearch;
-import cn.reghao.tnb.content.api.iface.AdminVideoService;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import org.apache.dubbo.config.annotation.DubboReference;
-import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
-
-/**
- * @author reghao
- * @date 2024-02-15 09:36:39
- */
-@Api(tags = "视频管理接口")
-@RestController
-@RequestMapping("/api/admin/video")
-public class ContentController {
-    @DubboReference(check = false, retries = 0, timeout = 10_000)
-    private AdminVideoService adminVideoService;
-
-    @ApiOperation("获取稿件数据")
-    @GetMapping(value = "/post", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String getPostList(VideoSearch videoSearch) {
-        PageList<UserVideoPost> pageList = adminVideoService.getVideoPosts(videoSearch);
-        return WebResult.success(pageList);
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/controller/GatewayController.java

@@ -1,49 +0,0 @@
-package cn.reghao.bnt.web.admin.controller;
-
-import cn.reghao.jutil.jdk.result.WebResult;
-import cn.reghao.bnt.web.admin.model.dto.NoticeContent;
-import cn.reghao.bnt.web.admin.service.SiteService;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import org.springframework.http.MediaType;
-import org.springframework.validation.annotation.Validated;
-import org.springframework.web.bind.annotation.*;
-
-/**
- * @author reghao
- * @date 2024-02-15 09:28:09
- */
-@Api(tags = "站点管理接口")
-@RestController
-@RequestMapping("/api/gateway")
-public class GatewayController {
-    private final SiteService siteService;
-
-    public GatewayController(SiteService siteService) {
-        this.siteService = siteService;
-    }
-
-    @ApiOperation("添加站点公告")
-    @PostMapping(value = "/site/notice", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String addSiteNotice(@RequestBody @Validated NoticeContent noticeContent) {
-        siteService.add(noticeContent.getContent());
-        return WebResult.success();
-    }
-
-    @ApiOperation("获取站点公告")
-    @GetMapping(value = "/site/notice", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String getSiteNotice() {
-        String content = siteService.get();
-        return WebResult.success(content);
-    }
-
-    @GetMapping(value = "/message", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String getMessages() {
-        return WebResult.success();
-    }
-
-    @PostMapping(value = "/message/approve", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String approve() {
-        return WebResult.success();
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/controller/MessageController.java

@@ -1,47 +0,0 @@
-package cn.reghao.bnt.web.admin.controller;
-
-import cn.reghao.jutil.jdk.result.Result;
-import cn.reghao.jutil.jdk.result.WebResult;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.*;
-
-import javax.validation.Valid;
-import java.util.List;
-
-/**
- * @author reghao
- * @date 2019-08-30 18:49:15
- */
-@Slf4j
-@Api(tags = "机器接口")
-@RestController
-@RequestMapping("/api/message")
-public class MessageController {
-    @ApiOperation(value = "设置机器环境")
-    @PostMapping(value = "/env/{machineId}", produces = MediaType.APPLICATION_JSON_VALUE)
-    public ResponseEntity<String> editExtraMachineInfo(@PathVariable("machineId") String machineId,
-                                                       @RequestParam("env") String env) {
-        return ResponseEntity.ok().body(WebResult.success());
-    }
-
-    @ApiOperation(value = "编辑 SSH 认证信息")
-    @PostMapping(value = "/sshauth/{machineId}", produces = MediaType.APPLICATION_JSON_VALUE)
-    public ResponseEntity<String> editSshAuth(@PathVariable("machineId") String machineId) {
-        return ResponseEntity.ok().body(WebResult.success());
-    }
-
-    @ApiOperation(value = "删除机器")
-    @DeleteMapping(value = "/{machineId}", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String machine(@PathVariable("machineId") String machineId) {
-        return WebResult.result(Result.success());
-    }
-
-    @PostMapping(value = "/delete", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String deleteAll(@RequestParam(value = "ids") List<String> machineIds) {
-        return WebResult.result(Result.success());
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/controller/OssFileController.java

@@ -1,64 +0,0 @@
-package cn.reghao.bnt.web.admin.controller;
-
-import cn.reghao.jutil.jdk.result.Result;
-import cn.reghao.jutil.jdk.result.WebResult;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.http.MediaType;
-import org.springframework.validation.annotation.Validated;
-import org.springframework.web.bind.annotation.*;
-
-import java.util.List;
-
-/**
- * @author reghao
- * @date 2019-11-27 11:29:43
- */
-@Slf4j
-@Api(tags = "应用配置 CRUD 接口")
-@RestController
-@RequestMapping("/api/oss/config/app")
-public class OssFileController {
-    @ApiOperation(value = "添加应用配置")
-    @PostMapping(produces = MediaType.APPLICATION_JSON_VALUE)
-    public String addAppConfig() {
-        Result result = Result.success();
-        return WebResult.result(result);
-    }
-
-    // TODO 使用 @PathVariable 注解时会自动填充实体
-    @ApiOperation(value = "复制应用配置")
-    @PostMapping(value = "/copy", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String copyAppConfig() {
-        Result result = Result.success();
-        return WebResult.result(result);
-    }
-
-    @ApiOperation(value = "修改应用配置")
-    @PostMapping(value = "/update", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String updateAppConfig() {
-        Result result = Result.success();
-        return WebResult.result(result);
-    }
-
-    @ApiOperation(value = "删除应用配置")
-    @DeleteMapping(value = "/{id}", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String deleteAppConfig(@PathVariable("id") String app) {
-        Result result = Result.success();
-        return WebResult.result(result);
-    }
-
-    @ApiOperation(value = "清空应用本地仓库")
-    @DeleteMapping(value = "/repo/{id}", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String deleteAppLocalRepo(@PathVariable("id") String app) {
-        Result result = Result.success();
-        return WebResult.result(result);
-    }
-
-    @PostMapping(value = "/enable", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String deleteAll(@RequestParam(value = "ids") List<String> appIds) {
-        Result result = Result.success();
-        return WebResult.result(result);
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/controller/SiteController.java

@@ -1,54 +0,0 @@
-package cn.reghao.bnt.web.admin.controller;
-
-import cn.reghao.jutil.jdk.result.WebResult;
-import cn.reghao.bnt.admin.api.dto.AdminMessage;
-import cn.reghao.bnt.web.admin.model.dto.NoticeContent;
-import cn.reghao.bnt.web.admin.service.SiteService;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import org.springframework.http.MediaType;
-import org.springframework.validation.annotation.Validated;
-import org.springframework.web.bind.annotation.*;
-
-import java.util.List;
-
-/**
- * @author reghao
- * @date 2024-02-15 09:28:09
- */
-@Api(tags = "站点管理接口")
-@RestController
-@RequestMapping("/api/site")
-public class SiteController {
-    private final SiteService siteService;
-
-    public SiteController(SiteService siteService) {
-        this.siteService = siteService;
-    }
-
-    @ApiOperation("添加站点公告")
-    @PostMapping(value = "/notice", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String addSiteNotice(@RequestBody @Validated NoticeContent noticeContent) {
-        siteService.add(noticeContent.getContent());
-        return WebResult.success();
-    }
-
-    @ApiOperation("获取站点公告")
-    @GetMapping(value = "/notice", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String getSiteNotice() {
-        String content = siteService.get();
-        return WebResult.success(content);
-    }
-
-    @GetMapping(value = "/message", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String getMessages() {
-        List<AdminMessage> list = siteService.getChargeReqs();
-        return WebResult.success(list);
-    }
-
-    @PostMapping(value = "/message/approve", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String approve(@RequestBody AdminMessage chargeReq) {
-        siteService.approve();
-        return WebResult.success();
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/controller/page/AccountPageController.java

@@ -1,64 +0,0 @@
-package cn.reghao.bnt.web.admin.controller.page;
-
-import cn.reghao.jutil.jdk.db.PageList;
-import cn.reghao.tnb.account.api.dto.AccountInfo;
-import cn.reghao.bnt.web.admin.service.AdminAccountService;
-import cn.reghao.tnb.user.api.dto.WalletChargeDto;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.data.domain.Page;
-import org.springframework.data.domain.PageImpl;
-import org.springframework.stereotype.Controller;
-import org.springframework.ui.Model;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-
-import java.util.*;
-
-/**
- * @author reghao
- * @date 2019-08-30 18:49:15
- */
-@Slf4j
-@Api(tags = "account 管理")
-@Controller
-@RequestMapping("/account")
-public class AccountPageController {
-    private final AdminAccountService accountService;
-
-    public AccountPageController(AdminAccountService accountService) {
-        this.accountService = accountService;
-    }
-
-    @ApiOperation(value = "帐号列表页面")
-    @GetMapping("/list")
-    public String indexPage(@RequestParam(value = "env", required = false) String env,
-                            @RequestParam(value = "machineIpv4", required = false) String machineIpv4,
-                            Model model) {
-        int pageNumber = 1;
-        int pageSize = 10;
-        PageList<AccountInfo> pageList = accountService.getUserAccounts(pageNumber);
-        long total = pageList.getTotalSize();
-
-        Page<AccountInfo> page = new PageImpl<>(pageList.getList());
-        model.addAttribute("env", env);
-        model.addAttribute("page", page);
-        model.addAttribute("list", page.getContent());
-        return "/admin/account/index";
-    }
-
-    @ApiOperation(value = "充值请求页面")
-    @GetMapping("/charge")
-    public String chargeReqPage(@RequestParam(value = "env", required = false) String env,
-                                @RequestParam(value = "machineIpv4", required = false) String machineIpv4,
-                                Model model) {
-
-        List<WalletChargeDto> list = accountService.getChargeReqs();
-        model.addAttribute("env", env);
-        model.addAttribute("page", Page.empty());
-        model.addAttribute("list", list);
-        return "/admin/account/chargeindex";
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/controller/page/ContentPageController.java

@@ -1,127 +0,0 @@
-package cn.reghao.bnt.web.admin.controller.page;
-
-import cn.reghao.bnt.web.util.DefaultSetting;
-import cn.reghao.jutil.jdk.db.PageList;
-import cn.reghao.bnt.web.admin.service.ContentService;
-import cn.reghao.tnb.content.api.dto.UserVideoPost;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.data.domain.Page;
-import org.springframework.data.domain.PageImpl;
-import org.springframework.stereotype.Controller;
-import org.springframework.ui.Model;
-import org.springframework.web.bind.annotation.*;
-
-import java.util.*;
-
-/**
- * @author reghao
- * @date 2019-08-30 18:49:15
- */
-@Slf4j
-@Api(tags = "content 管理")
-@Controller
-@RequestMapping("/content")
-public class ContentPageController {
-    private ContentService contentService;
-
-    public ContentPageController(ContentService contentService) {
-        this.contentService = contentService;
-    }
-
-    @ApiOperation(value = "视频分区")
-    @GetMapping("/category")
-    public String index(@RequestParam(value = "env", required = false) String env,
-                        @RequestParam(value = "type", required = false) String type,
-                        @RequestParam(value = "appName", required = false) String appName,
-                        Model model) {
-        if (env == null) {
-            env = DefaultSetting.getDefaultAppEnv();
-        }
-
-        if (type == null) {
-            type = DefaultSetting.getDefaultAppType();
-        }
-
-        model.addAttribute("env", env);
-        model.addAttribute("type", type);
-        model.addAttribute("page", Page.empty());
-        model.addAttribute("list", Collections.emptyList());
-        return "/admin/content/videocategory";
-    }
-
-    @ApiOperation(value = "稿件列表")
-    @GetMapping("/post")
-    public String postIndex(@RequestParam(value = "env", required = false) String env,
-                        @RequestParam(value = "type", required = false) String type,
-                        @RequestParam(value = "appName", required = false) String appName,
-                        Model model) {
-        if (env == null) {
-            env = DefaultSetting.getDefaultAppEnv();
-        }
-
-        if (type == null) {
-            type = DefaultSetting.getDefaultAppType();
-        }
-
-        PageList<UserVideoPost> pageList = contentService.getUserVideoPosts(1);
-        Page<UserVideoPost> page = new PageImpl<>(pageList.getList());
-
-        model.addAttribute("env", env);
-        model.addAttribute("type", type);
-        model.addAttribute("page", page);
-        model.addAttribute("list", page.getContent());
-        return "/admin/content/videolist";
-    }
-
-    @ApiOperation(value = "推荐模块")
-    @GetMapping("/video/preview/{videoId}")
-    public String previewPage(@PathVariable(value = "videoId") String videoId, Model model) {
-
-        model.addAttribute("videoInfo", "type");
-        return "/admin/content/videopreview";
-    }
-
-    @ApiOperation(value = "推荐模块")
-    @GetMapping("/rcmd")
-    public String rcmdIndex(@RequestParam(value = "env", required = false) String env,
-                            @RequestParam(value = "type", required = false) String type,
-                            @RequestParam(value = "appName", required = false) String appName,
-                            Model model) {
-        if (env == null) {
-            env = DefaultSetting.getDefaultAppEnv();
-        }
-
-        if (type == null) {
-            type = DefaultSetting.getDefaultAppType();
-        }
-
-        model.addAttribute("env", env);
-        model.addAttribute("type", type);
-        model.addAttribute("page", Page.empty());
-        model.addAttribute("list", Collections.emptyList());
-        return "/devops/app/rcmdindex";
-    }
-
-    @ApiOperation(value = "搜索模块")
-    @GetMapping("/search")
-    public String searchIndex(@RequestParam(value = "env", required = false) String env,
-                              @RequestParam(value = "type", required = false) String type,
-                              @RequestParam(value = "appName", required = false) String appName,
-                              Model model) {
-        if (env == null) {
-            env = DefaultSetting.getDefaultAppEnv();
-        }
-
-        if (type == null) {
-            type = DefaultSetting.getDefaultAppType();
-        }
-
-        model.addAttribute("env", env);
-        model.addAttribute("type", type);
-        model.addAttribute("page", Page.empty());
-        model.addAttribute("list", Collections.emptyList());
-        return "/devops/app/searchindex";
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/controller/page/FilePageController.java

@@ -1,114 +0,0 @@
-package cn.reghao.bnt.web.admin.controller.page;
-
-import cn.reghao.bnt.web.account.service.UserContext;
-import cn.reghao.jutil.jdk.result.Result;
-import cn.reghao.jutil.jdk.result.WebResult;
-import cn.reghao.oss.api.constant.ObjectScope;
-import cn.reghao.oss.api.constant.ObjectType;
-import cn.reghao.oss.api.dto.ChannelScopeDto;
-import cn.reghao.oss.api.dto.ObjectChannel;
-import cn.reghao.oss.api.dto.UploadChannelDto;
-import cn.reghao.oss.api.dto.UploadChannelVo;
-import cn.reghao.oss.api.iface.ObjectChannelService;
-import cn.reghao.bnt.web.admin.model.vo.KeyValue;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.http.MediaType;
-import org.springframework.stereotype.Controller;
-import org.springframework.ui.Model;
-import org.springframework.validation.annotation.Validated;
-import org.springframework.web.bind.annotation.*;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * @author reghao
- * @date 2019-08-30 18:49:15
- */
-@Slf4j
-@Api(tags = "file 管理")
-@Controller
-@RequestMapping("/file/store/channel")
-public class FilePageController {
-    private final ObjectChannelService objectChannelService;
-
-    public FilePageController(ObjectChannelService objectChannelService) {
-        this.objectChannelService = objectChannelService;
-    }
-
-    @ApiOperation(value = "上传通道列表页面")
-    @GetMapping(value = "/list")
-    public String index(@RequestParam(value = "env", required = false) String env,
-                        @RequestParam(value = "type", required = false) String type,
-                        @RequestParam(value = "appName", required = false) String appName, Model model) throws Exception {
-        int loginUser = UserContext.getUser().getId();
-        List<UploadChannelVo> list = objectChannelService.getObjectChannels(loginUser);
-        model.addAttribute("list", list);
-        return "/admin/channel/index";
-    }
-
-    @ApiOperation(value = "新增上传通道页面")
-    @GetMapping("/add")
-    public String addChannelPage(Model model) {
-        List<KeyValue> objectTypes = new ArrayList<>();
-        for (ObjectType objectType : ObjectType.values()) {
-            if (objectType.getCode() == 1000) {
-                continue;
-            }
-            objectTypes.add(new KeyValue(objectType.getCode()+"", objectType.name()));
-        }
-
-        List<KeyValue> objectScopes = new ArrayList<>();
-        for (ObjectScope objectScope : ObjectScope.values()) {
-            objectScopes.add(new KeyValue(objectScope.getCode()+"", objectScope.name()));
-        }
-
-        List<KeyValue> sizeList = new ArrayList<>();
-        sizeList.add(new KeyValue(1024L*1024*2+"", "2MB"));
-        sizeList.add(new KeyValue(1024L*1024*10+"", "10MB"));
-        sizeList.add(new KeyValue(1024L*1024*100+"", "100MB"));
-        sizeList.add(new KeyValue(1024L*1024*1024+"", "1GB"));
-        sizeList.add(new KeyValue(1024L*1024*1024*10+"", "10GB"));
-        sizeList.add(new KeyValue(1024L*1024*1024*20+"", "20GB"));
-
-        model.addAttribute("objectTypes", objectTypes);
-        model.addAttribute("objectScopes", objectScopes);
-        model.addAttribute("sizeList", sizeList);
-        return "/admin/channel/add";
-    }
-
-    @ApiOperation(value = "添加上传通道")
-    @PostMapping(value = "/add", produces = MediaType.APPLICATION_JSON_VALUE)
-    @ResponseBody
-    public String addUploadChannel(@Validated UploadChannelDto uploadChannelDto) {
-        Result result = objectChannelService.addObjectChannel(uploadChannelDto);
-        return WebResult.result(result);
-    }
-
-    @ApiOperation(value = "修改通道可见范围")
-    @GetMapping("/edit/{id}")
-    public String editChannelScopePage(@PathVariable("id") Integer id, Model model) {
-        ObjectChannel objectChannel = objectChannelService.getObjectChannel(id);
-        int scope = objectChannel.getScope();
-        ObjectScope objectScope0 = ObjectScope.getByCode(scope);
-
-        List<KeyValue> objectScopes = new ArrayList<>();
-        for (ObjectScope objectScope : ObjectScope.values()) {
-            objectScopes.add(new KeyValue(objectScope.getCode()+"", objectScope.name()));
-        }
-
-        model.addAttribute("objectScope", objectScope0.getName());
-        model.addAttribute("objectScopes", objectScopes);
-        return "/admin/channel/edit";
-    }
-
-    @ApiOperation(value = "修改通道可见范围")
-    @PostMapping(value = "/update/scope", produces = MediaType.APPLICATION_JSON_VALUE)
-    @ResponseBody
-    public String updateChannelScope(ChannelScopeDto channelScopeDto) {
-        objectChannelService.updateScope(channelScopeDto);
-        return WebResult.success();
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/controller/page/GatewayPageController.java

@@ -1,30 +0,0 @@
-package cn.reghao.bnt.web.admin.controller.page;
-
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.data.domain.Page;
-import org.springframework.stereotype.Controller;
-import org.springframework.ui.Model;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-
-import java.util.Collections;
-
-/**
- * @author reghao
- * @date 2019-08-30 18:49:15
- */
-@Slf4j
-@Api(tags = "gateway 管理")
-@Controller
-@RequestMapping("/gateway")
-public class GatewayPageController {
-    @ApiOperation(value = "路由列表页面")
-    @GetMapping("/route")
-    public String buildDirPage(Model model) {
-        model.addAttribute("page", Page.empty());
-        model.addAttribute("list", Collections.emptyList());
-        return "/devops/app/config/build/builddir";
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/controller/page/MessagePageController.java

@@ -1,30 +0,0 @@
-package cn.reghao.bnt.web.admin.controller.page;
-
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.data.domain.Page;
-import org.springframework.stereotype.Controller;
-import org.springframework.ui.Model;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-
-import java.util.Collections;
-
-/**
- * @author reghao
- * @date 2019-08-30 18:49:15
- */
-@Slf4j
-@Api(tags = "message 管理")
-@Controller
-@RequestMapping("/message")
-public class MessagePageController {
-    @ApiOperation(value = "消息列表页面")
-    @GetMapping("/list")
-    public String buildDirPage(Model model) {
-        model.addAttribute("page", Page.empty());
-        model.addAttribute("list", Collections.emptyList());
-        return "/devops/app/config/build/builddir";
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/controller/page/SitePageController.java

@@ -1,33 +0,0 @@
-package cn.reghao.bnt.web.admin.controller.page;
-
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.data.domain.Page;
-import org.springframework.stereotype.Controller;
-import org.springframework.ui.Model;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-
-import java.util.Collections;
-
-/**
- * @author reghao
- * @date 2019-08-30 18:49:15
- */
-@Slf4j
-@Api(tags = "site 管理")
-@Controller
-@RequestMapping("/site")
-public class SitePageController {
-    @ApiOperation(value = "站点配置页面")
-    @GetMapping("/config")
-    public String indexPage(@RequestParam(value = "env", required = false) String env,
-                            Model model) {
-        model.addAttribute("env", env);
-        model.addAttribute("page", Page.empty());
-        model.addAttribute("list", Collections.emptyList());
-        return "/admin/site/config";
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/model/constant/AppType.java

@@ -1,15 +0,0 @@
-package cn.reghao.bnt.web.admin.model.constant;
-
-/**
- * 应用类型列表
- *
- * @author reghao
- * @date 2019-10-18 14:31:29
- */
-public enum AppType {
-    maven, dotnetCore, npm;
-
-    public String getName() {
-        return this.name();
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/model/constant/BuildStatus.java

@@ -1,30 +0,0 @@
-package cn.reghao.bnt.web.admin.model.constant;
-
-/**
- * 构建状态
- *
- * @author reghao
- * @date 2021-11-08 16:35:42
- */
-public enum BuildStatus {
-    neverBuild(1, "尚未构建"),
-    onBuilding(2, "正在构建"),
-    buildSuccess(3, "构建成功"),
-    buildFail(4, "构建失败");
-
-    private final Integer code;
-    private final String desc;
-
-    BuildStatus(Integer code, String desc) {
-        this.code = code;
-        this.desc = desc;
-    }
-
-    public Integer getCode() {
-        return code;
-    }
-
-    public String getDesc() {
-        return desc;
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/model/constant/EnvType.java

@@ -1,19 +0,0 @@
-package cn.reghao.bnt.web.admin.model.constant;
-
-/**
- * @author reghao
- * @date 2023-12-01 14:17:44
- */
-public enum EnvType {
-    dev("开发环境"),test("测试环境"),uat("预发布环境"),prod("生产环境");
-
-    private final String desc;
-
-    EnvType(String desc) {
-        this.desc = desc;
-    }
-
-    public String getDesc() {
-        return desc;
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/model/constant/build/CompileType.java

@@ -1,15 +0,0 @@
-package cn.reghao.bnt.web.admin.model.constant.build;
-
-/**
- * 编译方式类型
- *
- * @author reghao
- * @date 2019-10-18 14:31:29
- */
-public enum CompileType {
-    none, shell, maven;
-
-    public String getName() {
-        return this.name();
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/model/constant/build/RepoAuthType.java

@@ -1,13 +0,0 @@
-package cn.reghao.bnt.web.admin.model.constant.build;
-
-/**
- * @author reghao
- * @date 2021-02-05 18:50:01
- */
-public enum RepoAuthType {
-    http, ssh, none;
-
-    public String getName() {
-        return this.name();
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/model/constant/build/RepoType.java

@@ -1,13 +0,0 @@
-package cn.reghao.bnt.web.admin.model.constant.build;
-
-/**
- * @author reghao
- * @date 2021-02-05 22:50:41
- */
-public enum RepoType {
-    git;
-
-    public String getName() {
-        return this.name();
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/model/dto/NoticeContent.java

@@ -1,15 +0,0 @@
-package cn.reghao.bnt.web.admin.model.dto;
-
-import lombok.Getter;
-
-import javax.validation.constraints.NotBlank;
-
-/**
- * @author reghao
- * @date 2023-12-07 14:36:32
- */
-@Getter
-public class NoticeContent {
-    @NotBlank
-    private String content;
-}

web/src/main/java/cn/reghao/bnt/web/admin/model/po/SiteNotice.java

@@ -1,17 +0,0 @@
-package cn.reghao.bnt.web.admin.model.po;
-
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-
-/**
- * @author reghao
- * @date 2023-12-07 14:03:51
- */
-@AllArgsConstructor
-@NoArgsConstructor
-@Getter
-public class SiteNotice {
-    private String content;
-    private long createBy;
-}

web/src/main/java/cn/reghao/bnt/web/admin/model/vo/KeyValue.java

@@ -1,15 +0,0 @@
-package cn.reghao.bnt.web.admin.model.vo;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-
-/**
- * @author reghao
- * @date 2021-06-03 19:00:57
- */
-@AllArgsConstructor
-@Data
-public class KeyValue {
-    private String key;
-    private String value;
-}

web/src/main/java/cn/reghao/bnt/web/admin/rpc/AdminServiceImpl.java

@@ -1,31 +0,0 @@
-package cn.reghao.bnt.web.admin.rpc;
-
-import cn.reghao.bnt.admin.api.dto.AdminMessage;
-import cn.reghao.bnt.admin.api.iface.AdminService;
-import cn.reghao.bnt.web.sys.db.repository.SysMessageRepository;
-import cn.reghao.bnt.web.sys.model.po.SysMessage;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.dubbo.config.annotation.DubboService;
-import org.springframework.stereotype.Service;
-
-/**
- * @author reghao
- * @date 2024-04-16 17:07:50
- */
-@Slf4j
-@DubboService
-@Service
-public class AdminServiceImpl implements AdminService {
-    private final SysMessageRepository sysMessageRepository;
-    
-    public AdminServiceImpl(SysMessageRepository sysMessageRepository) {
-        this.sysMessageRepository = sysMessageRepository;
-    }
-
-    @Override
-    public void sendMessage(AdminMessage adminMessage) {
-        log.info("新消息: {} -> {}", adminMessage.getType(), adminMessage.getContent());
-        SysMessage sysMessage = new SysMessage("充值消息", adminMessage.getContent());
-        sysMessageRepository.save(sysMessage);
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/rpc/SiteConfigServiceImpl.java

@@ -1,18 +0,0 @@
-package cn.reghao.bnt.web.admin.rpc;
-
-import cn.reghao.bnt.admin.api.iface.SiteConfigService;
-import org.apache.dubbo.config.annotation.DubboService;
-import org.springframework.stereotype.Service;
-
-/**
- * @author reghao
- * @date 2024-04-06 20:41:11
- */
-@DubboService
-@Service
-public class SiteConfigServiceImpl implements SiteConfigService {
-    @Override
-    public String getSiteNotice() {
-        return null;
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/service/AdminAccountService.java

@@ -1,40 +0,0 @@
-package cn.reghao.bnt.web.admin.service;
-
-import cn.reghao.jutil.jdk.db.PageList;
-import cn.reghao.tnb.account.api.dto.AccountInfo;
-import cn.reghao.tnb.account.api.iface.AccountQuery;
-import cn.reghao.tnb.user.api.dto.WalletChargeDto;
-import cn.reghao.tnb.user.api.iface.UserWalletService;
-import org.apache.dubbo.config.annotation.DubboReference;
-import org.springframework.stereotype.Service;
-
-import java.util.List;
-
-/**
- * @author reghao
- * @date 2024-07-06 11:22:17
- */
-@Service
-public class AdminAccountService {
-    @DubboReference(check = false)
-    private AccountQuery accountQuery;
-    @DubboReference(check = false)
-    private UserWalletService userWalletService;
-
-    public PageList<AccountInfo> getUserAccounts(int pageNumber) {
-        String screenName = "";
-        return accountQuery.getByScreenName(screenName, pageNumber);
-    }
-
-    public List<WalletChargeDto> getChargeReqs() {
-        return userWalletService.getChargeReqs();
-    }
-
-    public void approve(long chargeId) {
-        userWalletService.approveCharge(chargeId);
-    }
-
-    public void decline(long chargeId) {
-        userWalletService.declineCharge(chargeId);
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/service/ContentService.java

@@ -1,26 +0,0 @@
-package cn.reghao.bnt.web.admin.service;
-
-import cn.reghao.jutil.jdk.db.PageList;
-import cn.reghao.tnb.content.api.dto.UserVideoPost;
-import cn.reghao.tnb.content.api.dto.VideoSearch;
-import cn.reghao.tnb.content.api.iface.AdminVideoService;
-import org.apache.dubbo.config.annotation.DubboReference;
-import org.springframework.stereotype.Service;
-
-/**
- * @author reghao
- * @date 2024-07-06 22:16:55
- */
-@Service
-public class ContentService {
-    @DubboReference(check = false)
-    private AdminVideoService adminVideoService;
-
-    public PageList<UserVideoPost> getUserVideoPosts(int pageNumber) {
-        String title = "";
-        VideoSearch videoSearch = new VideoSearch();
-        videoSearch.setType(1);
-        videoSearch.setPage(pageNumber);
-        return adminVideoService.getVideoPosts(videoSearch);
-    }
-}

web/src/main/java/cn/reghao/bnt/web/admin/service/SiteService.java

@@ -1,36 +0,0 @@
-package cn.reghao.bnt.web.admin.service;
-
-import cn.reghao.bnt.admin.api.dto.AdminMessage;
-import cn.reghao.bnt.admin.api.iface.AdminService;
-import cn.reghao.bnt.web.admin.model.po.SiteNotice;
-import org.apache.dubbo.config.annotation.DubboReference;
-import org.springframework.stereotype.Service;
-
-import java.util.Collections;
-import java.util.List;
-
-/**
- * @author reghao
- * @date 2023-12-07 14:10:09
- */
-@Service
-public class SiteService {
-    @DubboReference(check = false)
-    private AdminService chargeService;
-
-    public void add(String content) {
-        long createBy = 10000;
-        SiteNotice siteNotice = new SiteNotice(content, createBy);
-    }
-
-    public String get() {
-        return null;
-    }
-
-    public List<AdminMessage> getChargeReqs() {
-        return Collections.emptyList();
-    }
-
-    public void approve() {
-    }
-}

web/src/main/java/cn/reghao/bnt/web/blog/BlogController.java

@@ -1,8 +0,0 @@
-package cn.reghao.bnt.web.blog;
-
-/**
- * @author reghao
- * @date 2024-07-24 13:50:19
- */
-public class BlogController {
-}

web/src/main/java/cn/reghao/bnt/web/config/spring/BeansConfig.java

@@ -1,12 +1,8 @@
 package cn.reghao.bnt.web.config.spring;
 
-import cn.reghao.bnt.web.config.AppProperties;
-import cn.reghao.bnt.web.parser.util.FileCache;
 import cn.reghao.jutil.jdk.converter.ByteConverter;
 import cn.reghao.jutil.jdk.http.WebClient;
 import cn.reghao.jutil.jdk.http.WebRequest;
-import cn.reghao.jutil.tool.http.DefaultWebRequest;
-import cn.reghao.oss.sdk.model.OssConsoleConfig;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import oshi.SystemInfo;

web/src/main/java/cn/reghao/bnt/web/console/config/ConsoleConfig.java

@@ -1,20 +0,0 @@
-package cn.reghao.bnt.web.console.config;
-
-import cn.reghao.oss.sdk.model.OssConsoleConfig;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-
-/**
- * @author reghao
- * @date 2024-07-29 16:47:14
- */
-@Configuration
-public class ConsoleConfig {
-    @Bean
-    public OssConsoleConfig ossConsoleConfig() {
-        String endpoint = System.getenv("CONSOLE_ENDPOINT");
-        String accessKeyId = System.getenv("CONSOLE_ACCESS_KEY_ID");
-        String accessKeySecret = System.getenv("CONSOLE_ACCESS_KEY_SECRET");
-        return new OssConsoleConfig(endpoint, accessKeyId, accessKeySecret);
-    }
-}

web/src/main/java/cn/reghao/bnt/web/console/config/OssConsoleClientFactory.java

@@ -1,37 +0,0 @@
-package cn.reghao.bnt.web.console.config;
-
-import cn.reghao.oss.sdk.OssConsoleClient;
-import cn.reghao.oss.sdk.model.OssConsoleConfig;
-import org.springframework.stereotype.Component;
-
-/**
- * @author reghao
- * @date 2024-02-24 17:41:42
- */
-@Component
-public class OssConsoleClientFactory {
-    private OssConsoleClient ossConsoleClient;
-    private final OssConsoleConfig ossConsoleConfig;
-
-    public OssConsoleClientFactory(OssConsoleConfig ossConsoleConfig) {
-        this.ossConsoleConfig = ossConsoleConfig;
-    }
-
-    public OssConsoleClient getOssConsoleClient() {
-        if (ossConsoleClient == null) {
-            ossConsoleClient = get();
-        }
-
-        return ossConsoleClient;
-    }
-
-    private OssConsoleClient get() {
-        try {
-            return new OssConsoleClient(ossConsoleConfig);
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-
-        return null;
-    }
-}

web/src/main/java/cn/reghao/bnt/web/console/controller/KeyAuthController.java

@@ -1,39 +0,0 @@
-package cn.reghao.bnt.web.console.controller;
-
-import cn.reghao.jutil.jdk.result.Result;
-import cn.reghao.jutil.jdk.result.WebResult;
-import cn.reghao.bnt.web.console.model.dto.KeyAuthDto;
-import cn.reghao.bnt.web.console.service.UserKeyService;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import org.springframework.http.MediaType;
-import org.springframework.validation.annotation.Validated;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
-
-/**
- * @author reghao
- * @date 2024-02-27 11:32:23
- */
-@Api(tags = "oss key 认证接口")
-@RestController
-@RequestMapping("/api/oss/key")
-public class KeyAuthController {
-    private final UserKeyService userKeyService;
-
-    public KeyAuthController(UserKeyService userKeyService) {
-        this.userKeyService = userKeyService;
-    }
-
-    @ApiOperation(value = "认证 oss key")
-    @PostMapping(value = "/auth", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String authKey(@Validated KeyAuthDto keyAuthDto) {
-        Result result = userKeyService.auth(keyAuthDto);
-        if (result.getCode() == 0) {
-            return WebResult.success(result.getData());
-        }
-
-        return WebResult.result(result);
-    }
-}

web/src/main/java/cn/reghao/bnt/web/console/controller/OssController.java

@@ -1,115 +0,0 @@
-package cn.reghao.bnt.web.console.controller;
-
-import cn.reghao.jutil.jdk.result.WebResult;
-import cn.reghao.oss.api.dto.ObjectInfo;
-import cn.reghao.oss.api.dto.ServerInfo;
-import cn.reghao.oss.api.iface.ConsoleService;
-import cn.reghao.oss.api.iface.StoreServiceWrapper;
-import cn.reghao.oss.api.util.AuthContext;
-import cn.reghao.oss.api.dto.media.*;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.*;
-
-/**
- * @author reghao
- * @date 2024-02-23 09:05:58
- */
-@Api(tags = "视频文件接口")
-@RestController
-@RequestMapping("/api/oss")
-public class OssController {
-    private final ConsoleService consoleService;
-    private final StoreServiceWrapper storeServiceWrapper;
-
-    public OssController(ConsoleService consoleService, StoreServiceWrapper storeServiceWrapper) {
-        this.consoleService = consoleService;
-        this.storeServiceWrapper = storeServiceWrapper;
-    }
-
-    @ApiOperation(value = "获取 oss-store 节点")
-    @GetMapping(value = "/upload/store", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String getUploadStore(@RequestParam("channelId") int channelId) {
-        int loginUser = AuthContext.getUserId();
-        ServerInfo serverInfo = consoleService.getUploadStore(loginUser, channelId);
-        return WebResult.success(serverInfo);
-    }
-
-    @ApiOperation(value = "获取 channel 可见范围")
-    @GetMapping(value = "/channel/scope", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String getChannelScope(int channelId) throws Exception {
-        int scope = storeServiceWrapper.getChannelScope(channelId);
-        return WebResult.success(scope);
-    }
-
-    @ApiOperation(value = "设置对象可见范围")
-    @PostMapping(value = "/object/scope", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String setObjectScope(int channelId, String objectId, int scope) {
-        storeServiceWrapper.setObjectScope(channelId, objectId, scope);
-        return WebResult.success();
-    }
-
-    @ApiOperation(value = "根据 object_id 删除对象")
-    @PostMapping(value = "/object/delete/id", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String deleteByObjectId(@RequestParam("channelId") int channelId,
-                                   @RequestParam("objectId") String objectId) {
-        storeServiceWrapper.deleteByObjectId(channelId, objectId);
-        return WebResult.success();
-    }
-
-    @ApiOperation(value = "根据 object_url 删除对象")
-    @PostMapping(value = "/object/delete/url", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String deleteByObjectUrl(@RequestParam("objectUrl") String objectUrl) {
-        storeServiceWrapper.deleteByObjectUrl(objectUrl);
-        return WebResult.success();
-    }
-
-    @ApiOperation(value = "获取对象信息")
-    @GetMapping(value = "/object/info", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String getObjectInfo(@RequestParam("channelId") int channelId,
-                                @RequestParam("objectId") String objectId) throws Exception {
-        ObjectInfo objectInfo = storeServiceWrapper.getObjectInfo(channelId, objectId);
-        return WebResult.success(objectInfo);
-    }
-
-    @ApiOperation(value = "获取对象签名 url")
-    @GetMapping(value = "/object/url", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String getSignedUrl(@RequestParam("channelId") Integer channelId,
-                               @RequestParam("objectId") String objectId) throws Exception {
-        String signedUrl = storeServiceWrapper.getSignedUrl(channelId, objectId);
-        return WebResult.success(signedUrl);
-    }
-
-    @ApiOperation(value = "获取视频文件信息")
-    @GetMapping(value = "/object/video/info", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String getVideoInfo(@RequestParam("channelId") Integer channelId,
-                               @RequestParam("objectId") String objectId) throws Exception {
-        VideoInfo videoInfo = storeServiceWrapper.getVideoInfo(channelId, objectId);
-        return WebResult.success(videoInfo);
-    }
-
-    @ApiOperation(value = "获取图片文件信息")
-    @GetMapping(value = "/object/image/info", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String getImagesInfo(@RequestParam("channelId") Integer channelId,
-                                @RequestParam("objectId") String objectId) throws Exception {
-        ImageInfo imageInfo = storeServiceWrapper.getImageInfo(channelId, objectId);
-        return WebResult.success(imageInfo);
-    }
-
-    @ApiOperation(value = "获取 webp 图片文件信息")
-    @PostMapping(value = "/object/image/webp", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String getWebpInfo(@RequestParam("channelId") Integer channelId,
-                              @RequestParam("objectId") String objectId) throws Exception {
-        ConvertedImageInfo convertedImageInfo = storeServiceWrapper.getWebpInfo(channelId, objectId);
-        return WebResult.success(convertedImageInfo);
-    }
-
-    @ApiOperation(value = "获取音频文件信息")
-    @GetMapping(value = "/media/audio/info", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String getAudioInfo(@RequestParam("channelId") Integer channelId,
-                               @RequestParam("objectId") String objectId) throws Exception {
-        AudioInfo audioInfo = storeServiceWrapper.getAudioInfo(channelId, objectId);
-        return WebResult.success(audioInfo);
-    }
-}

web/src/main/java/cn/reghao/bnt/web/console/controller/page/StoreNodeController.java

@@ -1,42 +0,0 @@
-package cn.reghao.bnt.web.console.controller.page;
-
-import cn.reghao.jutil.jdk.result.Result;
-import cn.reghao.bnt.web.console.model.dto.NodeUpdateDto;
-import cn.reghao.jutil.jdk.result.WebResult;
-import cn.reghao.bnt.web.console.service.StoreNodeService;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.http.MediaType;
-import org.springframework.validation.annotation.Validated;
-import org.springframework.web.bind.annotation.*;
-
-/**
- * @author reghao
- * @date 2024-02-23 11:29:14
- */
-@Slf4j
-@Api(tags = "存储节点数据接口")
-@RestController
-@RequestMapping("/api/store/node")
-public class StoreNodeController {
-    private final StoreNodeService storeNodeService;
-
-    public StoreNodeController(StoreNodeService storeNodeService) {
-        this.storeNodeService = storeNodeService;
-    }
-
-    @ApiOperation(value = "设置存储节点的状态")
-    @PostMapping(value = "/update", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String updateStoreNodeDomain() {
-        storeNodeService.updateStatus();
-        return WebResult.success();
-    }
-
-    @ApiOperation(value = "删除存储节点")
-    @DeleteMapping(value = "/{id}", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String deleteStoreNode(@PathVariable("id") Integer nodeId) {
-        Result result = storeNodeService.delete(nodeId);
-        return WebResult.result(result);
-    }
-}

web/src/main/java/cn/reghao/bnt/web/console/controller/page/StoreNodePageController.java

@@ -1,76 +0,0 @@
-package cn.reghao.bnt.web.console.controller.page;
-
-import cn.reghao.bnt.web.console.model.po.StoreNode;
-import cn.reghao.bnt.web.console.model.vo.StoreNodeVO;
-import cn.reghao.bnt.web.console.service.StoreChannelService;
-import cn.reghao.bnt.web.console.service.StoreNodeService;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import org.springframework.stereotype.Controller;
-import org.springframework.ui.Model;
-import org.springframework.web.bind.annotation.*;
-
-import java.util.*;
-
-/**
- * @author reghao
- * @date 2024-02-23 11:29:14
- */
-@Api(tags = "存储节点页面")
-@Controller
-@RequestMapping("/store/node")
-public class StoreNodePageController {
-    private final StoreNodeService storeNodeService;
-    private final StoreChannelService storeChannelService;
-
-    public StoreNodePageController(StoreNodeService storeNodeService, StoreChannelService storeChannelService) {
-        this.storeNodeService = storeNodeService;
-        this.storeChannelService = storeChannelService;
-    }
-
-    @ApiOperation(value = "存储节点列表页面")
-    @GetMapping("/list")
-    public String storeNodePage(@RequestParam(value = "env", required = false) String env,
-                                @RequestParam(value = "type", required = false) String type,
-                                @RequestParam(value = "appName", required = false) String appName,
-                                Model model) {
-        if (env == null) {
-            env = "test";
-        }
-
-        if (type == null) {
-            type = "test";
-        }
-
-        if (appName != null) {
-            Map<String, String> map = new HashMap<>();
-            map.put("appName", appName);
-            //List<AppConfig> list = appConfigQuery.query(map);
-            List<String> list = Collections.emptyList();
-            //model.addAttribute("page", page);
-            model.addAttribute("list", list);
-            return "/console/node/index";
-        }
-
-        List<StoreNode> list = storeNodeService.getByPage();
-        //model.addAttribute("page", page);
-        model.addAttribute("list", list);
-        return "/console/node/index";
-    }
-
-    @ApiOperation(value = "存储节点域名设置页面")
-    @GetMapping("/config/{id}")
-    public String editStoreNodePage(@PathVariable("id") Integer id, Model model) {
-        StoreNode storeNode = storeNodeService.getById(id);
-        model.addAttribute("storeNode", storeNode);
-        return "/console/node/edit";
-    }
-
-    @ApiOperation(value = "存储节点详情页面")
-    @GetMapping("/detail/{id}")
-    public String storeNodePage(@PathVariable("id") Integer id, Model model) {
-        StoreNodeVO storeNode = storeChannelService.getStoreDetail(id);
-        model.addAttribute("storeNode", storeNode);
-        return "/console/node/detail";
-    }
-}

web/src/main/java/cn/reghao/bnt/web/console/controller/page/StoreObjectPageController.java

@@ -1,68 +0,0 @@
-package cn.reghao.bnt.web.console.controller.page;
-
-import cn.reghao.bnt.web.account.service.UserContext;
-import cn.reghao.jutil.jdk.result.WebResult;
-import cn.reghao.oss.api.dto.FileInfo;
-import cn.reghao.oss.api.iface.ConsoleService;
-import cn.reghao.oss.api.dto.ServerInfo;
-import cn.reghao.bnt.web.console.service.DiskFileService;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import org.springframework.stereotype.Controller;
-import org.springframework.ui.Model;
-import org.springframework.web.bind.annotation.*;
-import org.springframework.web.multipart.MultipartFile;
-
-import java.util.List;
-
-/**
- * @author reghao
- * @date 2024-02-27 09:12:14
- */
-@Api(tags = "存储对象页面")
-@Controller
-@RequestMapping("/store/object")
-public class StoreObjectPageController {
-    private final ConsoleService consoleService;
-    private DiskFileService diskFileService;
-
-    public StoreObjectPageController(ConsoleService consoleService, DiskFileService diskFileService) {
-        this.consoleService = consoleService;
-        this.diskFileService = diskFileService;
-    }
-
-    @ApiOperation(value = "文件列表页面")
-    @GetMapping(value = "/list")
-    public String index(@RequestParam(value = "env", required = false) String env,
-                        @RequestParam(value = "type", required = false) String type,
-                        @RequestParam(value = "appName", required = false) String appName, Model model) throws Exception {
-
-        int pageNumber = 1;
-        List<FileInfo> list = diskFileService.getFileList(pageNumber);
-
-        model.addAttribute("list", list);
-        return "/console/object/index";
-    }
-
-    @ApiOperation(value = "文件上传页面")
-    @GetMapping(value = "/upload")
-    public String uploadPage(Model model) {
-        int loginUser = UserContext.getUserId();
-        int channelId = 105;
-        try {
-            ServerInfo serverInfo = consoleService.getUploadStore(loginUser, channelId);
-            model.addAttribute("serverInfo", serverInfo);
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-
-        return "/console/object/upload";
-    }
-
-    @ApiOperation(value = "文件上传接口")
-    @PostMapping(value = "/upload")
-    @ResponseBody
-    public String uploadFile(MultipartFile[] files) {
-        return WebResult.success();
-    }
-}

web/src/main/java/cn/reghao/bnt/web/console/controller/page/UploadChannelController.java

@@ -1,43 +0,0 @@
-package cn.reghao.bnt.web.console.controller.page;
-
-import cn.reghao.jutil.jdk.result.Result;
-import cn.reghao.jutil.jdk.result.WebResult;
-import cn.reghao.oss.api.dto.ChannelScopeDto;
-import cn.reghao.oss.api.dto.UploadChannelDto;
-import cn.reghao.oss.api.iface.ObjectChannelService;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.http.MediaType;
-import org.springframework.validation.annotation.Validated;
-import org.springframework.web.bind.annotation.*;
-
-/**
- * @author reghao
- * @date 2024-02-23 15:05:30
- */
-@Slf4j
-@Api(tags = "上传通道接口")
-@RestController
-@RequestMapping("/api/store/channel")
-public class UploadChannelController {
-    private final ObjectChannelService objectChannelService;
-
-    public UploadChannelController(ObjectChannelService objectChannelService) {
-        this.objectChannelService = objectChannelService;
-    }
-
-    @ApiOperation(value = "添加上传通道")
-    @PostMapping(value = "/add", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String addUploadChannel(@Validated UploadChannelDto uploadChannelDto) {
-        Result result = objectChannelService.addObjectChannel(uploadChannelDto);
-        return WebResult.result(result);
-    }
-
-    @ApiOperation(value = "修改通道可见范围")
-    @PostMapping(value = "/update/scope", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String updateChannelScope(@Validated ChannelScopeDto channelScopeDto) {
-        objectChannelService.updateScope(channelScopeDto);
-        return WebResult.success();
-    }
-}

web/src/main/java/cn/reghao/bnt/web/console/controller/page/UploadChannelPageController.java

@@ -1,96 +0,0 @@
-package cn.reghao.bnt.web.console.controller.page;
-
-import cn.reghao.oss.api.constant.ObjectScope;
-import cn.reghao.oss.api.constant.ObjectType;
-import cn.reghao.oss.api.dto.ObjectChannel;
-import cn.reghao.oss.api.iface.ObjectChannelService;
-import cn.reghao.bnt.web.account.service.UserContext;
-import cn.reghao.bnt.web.console.model.vo.KeyValue;
-import cn.reghao.oss.api.dto.UploadChannelVo;
-import cn.reghao.oss.api.dto.UserNodeVO;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Controller;
-import org.springframework.ui.Model;
-import org.springframework.web.bind.annotation.*;
-
-import java.util.*;
-
-/**
- * @author reghao
- * @date 2024-02-23 15:05:30
- */
-@Slf4j
-@Api(tags = "上传通道页面")
-@Controller
-@RequestMapping("/store/channel")
-public class UploadChannelPageController {
-    private final ObjectChannelService objectChannelService;
-
-    public UploadChannelPageController(ObjectChannelService objectChannelService) {
-        this.objectChannelService = objectChannelService;
-    }
-
-    @ApiOperation(value = "上传通道列表页面")
-    @GetMapping(value = "/list")
-    public String index(@RequestParam(value = "env", required = false) String env,
-                        @RequestParam(value = "type", required = false) String type,
-                        @RequestParam(value = "appName", required = false) String appName, Model model) throws Exception {
-        int loginUser = UserContext.getUserId();
-        List<UploadChannelVo> list = objectChannelService.getObjectChannels(loginUser);
-        model.addAttribute("list", list);
-        return "/console/channel/index";
-    }
-
-    @ApiOperation(value = "新增上传通道页面")
-    @GetMapping("/add")
-    public String addChannelPage(Model model) {
-        List<KeyValue> objectTypes = new ArrayList<>();
-        for (ObjectType objectType : ObjectType.values()) {
-            if (objectType.getCode() == 1000) {
-                continue;
-            }
-            objectTypes.add(new KeyValue(objectType.getCode()+"", objectType.name()));
-        }
-
-        List<KeyValue> objectScopes = new ArrayList<>();
-        for (ObjectScope objectScope : ObjectScope.values()) {
-            objectScopes.add(new KeyValue(objectScope.getCode()+"", objectScope.name()));
-        }
-
-        List<KeyValue> sizeList = new ArrayList<>();
-        sizeList.add(new KeyValue(1024L*1024*2+"", "2MB"));
-        sizeList.add(new KeyValue(1024L*1024*10+"", "10MB"));
-        sizeList.add(new KeyValue(1024L*1024*100+"", "100MB"));
-        sizeList.add(new KeyValue(1024L*1024*1024+"", "1GB"));
-        sizeList.add(new KeyValue(1024L*1024*1024*10+"", "10GB"));
-        sizeList.add(new KeyValue(1024L*1024*1024*20+"", "20GB"));
-
-        int loginUser = UserContext.getUserId();
-        List<UserNodeVO> storeNodes = objectChannelService.getUserStoreNodes(loginUser);
-
-        model.addAttribute("objectTypes", objectTypes);
-        model.addAttribute("objectScopes", objectScopes);
-        model.addAttribute("sizeList", sizeList);
-        model.addAttribute("storeNodes", storeNodes);
-        return "/console/channel/add";
-    }
-
-    @ApiOperation(value = "修改通道可见范围")
-    @GetMapping("/edit/{id}")
-    public String editChannelScopePage(@PathVariable("id") Integer id, Model model) {
-        ObjectChannel objectChannel = objectChannelService.getObjectChannel(id);
-        int scope = objectChannel.getScope();
-        ObjectScope objectScope0 = ObjectScope.getByCode(scope);
-
-        List<KeyValue> objectScopes = new ArrayList<>();
-        for (ObjectScope objectScope : ObjectScope.values()) {
-            objectScopes.add(new KeyValue(objectScope.getCode()+"", objectScope.name()));
-        }
-
-        model.addAttribute("objectScope", objectScope0.getName());
-        model.addAttribute("objectScopes", objectScopes);
-        return "/console/channel/edit";
-    }
-}

web/src/main/java/cn/reghao/bnt/web/console/controller/page/UserKeyController.java

@@ -1,38 +0,0 @@
-package cn.reghao.bnt.web.console.controller.page;
-
-import cn.reghao.jutil.jdk.result.Result;
-import cn.reghao.jutil.jdk.result.WebResult;
-import cn.reghao.bnt.web.console.service.UserKeyService;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.*;
-
-/**
- * @author reghao
- * @date 2024-02-27 10:59:07
- */
-@Api(tags = "用户帐号接口")
-@RestController
-@RequestMapping("/api/store/my/key")
-public class UserKeyController {
-    private final UserKeyService userKeyService;
-
-    public UserKeyController(UserKeyService userKeyService) {
-        this.userKeyService = userKeyService;
-    }
-
-    @ApiOperation(value = "创建 oss key")
-    @PostMapping(value = "/create", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String create() {
-        Result result = userKeyService.create();
-        return WebResult.result(result);
-    }
-
-    @ApiOperation(value = "重新生成  oss key")
-    @PostMapping(value = "/regenerate", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String delete() {
-        userKeyService.regenerate();
-        return WebResult.success();
-    }
-}

web/src/main/java/cn/reghao/bnt/web/console/controller/page/UserKeyPageController.java

@@ -1,35 +0,0 @@
-package cn.reghao.bnt.web.console.controller.page;
-
-import cn.reghao.bnt.web.console.model.vo.UserKeyVo;
-import cn.reghao.bnt.web.console.service.UserKeyService;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import org.springframework.stereotype.Controller;
-import org.springframework.ui.Model;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-
-import java.util.List;
-
-/**
- * @author reghao
- * @date 2024-02-27 10:24:31
- */
-@Api(tags = "我的帐号页面")
-@Controller
-@RequestMapping("/store/my/key")
-public class UserKeyPageController {
-    private final UserKeyService userKeyService;
-
-    public UserKeyPageController(UserKeyService userKeyService) {
-        this.userKeyService = userKeyService;
-    }
-
-    @ApiOperation(value = "我的帐号页面")
-    @GetMapping(value = "")
-    public String userKeyPage(Model model) {
-        List<UserKeyVo> list = userKeyService.getUserKeys();
-        model.addAttribute("list", list);
-        return "/console/userkey/index1";
-    }
-}

web/src/main/java/cn/reghao/bnt/web/console/controller/page/UserNodeController.java

@@ -1,40 +0,0 @@
-package cn.reghao.bnt.web.console.controller.page;
-
-import cn.reghao.jutil.jdk.result.Result;
-import cn.reghao.jutil.jdk.result.WebResult;
-import cn.reghao.bnt.web.console.model.dto.UserNodeDto;
-import cn.reghao.bnt.web.console.service.UserNodeService;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import org.springframework.http.MediaType;
-import org.springframework.validation.annotation.Validated;
-import org.springframework.web.bind.annotation.*;
-
-/**
- * @author reghao
- * @date 2024-02-27 09:40:54
- */
-@Api(tags = "用户存储节点数据接口")
-@RestController
-@RequestMapping("/api/store/my")
-public class UserNodeController {
-    private final UserNodeService userNodeService;
-
-    public UserNodeController(UserNodeService userNodeService) {
-        this.userNodeService = userNodeService;
-    }
-
-    @ApiOperation(value = "添加用户节点")
-    @PostMapping(value = "/add", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String addUserNode(@Validated UserNodeDto userNodeDto) {
-        Result result = userNodeService.add(userNodeDto);
-        return WebResult.result(result);
-    }
-
-    @ApiOperation(value = "删除用户节点")
-    @DeleteMapping(value = "/delete/{id}", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String deleteUserNode(@PathVariable("id") Integer userNodeId) {
-        Result result = userNodeService.delete(userNodeId);
-        return WebResult.result(result);
-    }
-}