添加 nginx 日志统计功能

manager/pom.xml

@@ -44,6 +44,11 @@
             <artifactId>common</artifactId>
             <version>1.0.0-SNAPSHOT</version>
         </dependency>
+        <dependency>
+            <groupId>cn.reghao.devops</groupId>
+            <artifactId>model</artifactId>
+            <version>1.0.0-SNAPSHOT</version>
+        </dependency>
 
         <dependency>
             <groupId>org.springframework.boot</groupId>

manager/src/main/java/cn/reghao/devops/manager/app/controller/AppLogController.java

@@ -1,26 +1,51 @@
 package cn.reghao.devops.manager.app.controller;
 
+import cn.reghao.devops.manager.app.service.LoggingService;
+import cn.reghao.jutil.jdk.result.WebResult;
 import io.swagger.annotations.ApiOperation;
+import org.springframework.http.MediaType;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+import java.text.ParseException;
+import java.util.List;
 
 /**
  * @author reghao
  * @date 2023-11-02 13:39:54
  */
 @Controller
-@RequestMapping("/app/log")
 public class AppLogController {
+    private final LoggingService loggingService;
+
+    public AppLogController(LoggingService loggingService) {
+        this.loggingService = loggingService;
+    }
+
     @ApiOperation(value = "应用实时日志页面")
-    @GetMapping(value = "/online")
+    @GetMapping(value = "/app/log/online")
     public String appLogPage() {
         return "/app/stat/log";
     }
 
     @ApiOperation(value = "应用历史日志页面")
-    @GetMapping(value = "/offline")
+    @GetMapping(value = "/app/log/offline")
     public String appLog() {
         return "/app/stat/log1";
     }
+
+    @ApiOperation(value = "Nginx 日志页面")
+    @GetMapping(value = "/app/log/nginx")
+    public String nginxLog() {
+        return "/app/stat/nginxlog";
+    }
+
+    @GetMapping(value = "/api/log/nginx/chart", produces = MediaType.APPLICATION_JSON_VALUE)
+    @ResponseBody
+    public String nginxLogData() throws ParseException {
+        List list = loggingService.getChartData();
+        return WebResult.success(list);
+    }
 }

manager/src/main/java/cn/reghao/devops/manager/app/model/vo/ChartData.java

@@ -0,0 +1,17 @@
+package cn.reghao.devops.manager.app.model.vo;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import java.util.List;
+
+/**
+ * @author reghao
+ * @date 2023-11-07 11:10:45
+ */
+@Setter
+@Getter
+public class ChartData {
+    private List<String> xAxis;
+    private List<Integer> yAxis;
+}

manager/src/main/java/cn/reghao/devops/manager/app/service/LoggingService.java

@@ -0,0 +1,81 @@
+package cn.reghao.devops.manager.app.service;
+
+import cn.reghao.devops.model.NginxLog;
+import cn.reghao.jutil.jdk.converter.DateTimeConverter;
+import cn.reghao.jutil.jdk.serializer.JsonConverter;
+import cn.reghao.jutil.jdk.text.TextFile;
+import org.springframework.stereotype.Service;
+
+import java.text.ParseException;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.*;
+
+/**
+ * @author reghao
+ * @date 2023-11-08 10:08:02
+ */
+@Service
+public class LoggingService {
+    public List getChartData() throws ParseException {
+        TextFile textFile = new TextFile();
+        String filePath = "/home/reghao/Downloads/api.iquizoo.com.access.log-20231108";
+        List<String> list = textFile.read(filePath);
+
+        List<NginxLog> nginxLogs = new ArrayList<>();
+        for (String line : list) {
+            if (!line.startsWith("{")) {
+                continue;
+            }
+
+            try {
+                NginxLog nginxLog = JsonConverter.jsonToObject(line, NginxLog.class);
+                nginxLogs.add(nginxLog);
+            } catch (Exception e) {
+                // e.printStackTrace();
+            }
+        }
+
+        Map<Long, Integer> map = new TreeMap<>();
+        for (NginxLog nginxLog : nginxLogs) {
+            String date = nginxLog.getTimeIso8601();
+            DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss+08:00")
+                    .withZone(ZoneId.of("UTC"));
+            LocalDateTime localDateTime = LocalDateTime.parse(date, formatter);
+            Long timestamp = localDateTime.toEpochSecond(ZoneOffset.of("+8"));
+            Long key = timestamp;
+
+            Integer count = map.get(key);
+            if (count == null) {
+                map.put(key, 1);
+            } else {
+                int count1 = map.get(key) + 1;
+                map.put(key, count1);
+            }
+        }
+
+        LocalDateTime localDateTime = LocalDateTime.now();
+        Long baseKey = localDateTime.toEpochSecond(ZoneOffset.of("+8"));
+
+        List<String> xList = new ArrayList<>();
+        List<Integer> yList = new ArrayList<>();
+        Set<Long> keys = new HashSet<>();
+        for (Long key : map.keySet()) {
+            if (key < baseKey) {
+                //xList.add(DateTimeConverter.format(key*1000).split(" ")[1]);
+                xList.add(DateTimeConverter.format(key*1000));
+                yList.add(map.get(key));
+                keys.add(key);
+            }
+        }
+
+        keys.forEach(map::remove);
+        keys.clear();
+        List results = new ArrayList();
+        results.add(xList.toArray());
+        results.add(yList.toArray());
+        return results;
+    }
+}

manager/src/main/java/cn/reghao/devops/manager/home/controller/HomeController.java

@@ -23,7 +23,7 @@ import java.util.List;
  * @date 2022-09-23 11:24:18
  */
 @Slf4j
-@Api(tags = "首页统计信息接口")
+@Api(tags = "首页接口")
 @RequestMapping("/api/home")
 @RestController
 public class HomeController {
@@ -33,11 +33,6 @@ public class HomeController {
         this.appConfigRepository = appConfigRepository;
     }
 
-    @GetMapping(value = "/machine", produces = MediaType.APPLICATION_JSON_VALUE)
-    public String machineStatistic() {
-        return WebResult.success();
-    }
-
     @GetMapping(value = "/app", produces = MediaType.APPLICATION_JSON_VALUE)
     public String appStatistic(@RequestParam("env") String env) {
         PageRequest pageRequest = PageRequest.of(0, 1);

manager/src/main/java/cn/reghao/devops/manager/home/controller/page/HomePageController.java

@@ -1,32 +1,37 @@
 package cn.reghao.devops.manager.home.controller.page;
 
+import cn.reghao.devops.common.util.jvm.JVM;
+import cn.reghao.devops.common.util.jvm.po.JvmInfo;
+import cn.reghao.devops.common.util.jvm.po.JvmStat;
+import cn.reghao.devops.manager.home.service.HomeService;
 import cn.reghao.devops.manager.rbac.model.po.Menu;
 import cn.reghao.devops.manager.rbac.model.po.User;
-import cn.reghao.devops.manager.home.service.HomeService;
 import io.swagger.annotations.Api;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.boot.web.servlet.error.ErrorController;
+import io.swagger.annotations.ApiOperation;
 import org.springframework.security.core.Authentication;
 import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.stereotype.Controller;
 import org.springframework.ui.Model;
-import org.springframework.web.bind.annotation.*;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestParam;
 
-import javax.servlet.http.HttpServletRequest;
-import java.util.*;
+import java.util.List;
+import java.util.Map;
 
 /**
  * @author reghao
  * @date 2021-04-04 21:24:18
  */
-@Slf4j
-@Api(tags = "登录页和首页")
+@Api(tags = "home 页面")
 @Controller
-public class HomePageController implements ErrorController {
+public class HomePageController {
     private final HomeService homeService;
+    private final JVM jvm;
 
     public HomePageController(HomeService homeService) {
         this.homeService = homeService;
+        this.jvm = new JVM();
     }
 
     @GetMapping("/")
@@ -45,49 +50,38 @@ public class HomePageController implements ErrorController {
         return "/main";
     }
 
-    @GetMapping("/login")
-    public String toLogin(Model model) {
-        model.addAttribute("isCaptcha", false);
-        return "/login";
-    }
-
     @GetMapping("/home")
     public String home(@RequestParam(name = "env", required = false) String env, Model model) {
-        String defaultEnv = env != null ? env : "test";
-        model.addAttribute("env", defaultEnv);
+        JvmInfo jvmInfo = jvm.info();
+        String osInfo = String.format("%s %s", jvmInfo.getOsName(), jvmInfo.getOsVersion());
+        String jvmInfo1 = String.format("%s %s", jvmInfo.getJvmName(), jvmInfo.getJvmVersion());
+        int pid = jvmInfo.getJvmPid();
+        String startAt = jvmInfo.getJvmStartTime();
+        String processInfo = String.format("%s - %s", pid, startAt);
 
+        model.addAttribute("osInfo", osInfo);
+        model.addAttribute("jvmInfo", jvmInfo1);
+        model.addAttribute("pid", "pid: " + pid);
+        model.addAttribute("startAt", "启动时间: " + startAt);
         String template = "/home/index";
-        //template = "/home/index1";
+        template = "/home/index1";
         return template;
     }
 
-    @Override
-    public String getErrorPath() {
-        return "/error";
-    }
+    @ApiOperation(value = "JVM 状态页面")
+    @GetMapping("/sys/jvmstatus")
+    public String jvmStatusPage(Model model) {
+        JvmInfo jvmInfo = jvm.info();
+        JvmStat jvmStat = jvm.stat();
 
-    /**
-     * 处理错误页面
-     *
-     * @param
-     * @return
-     * @date 2021-05-19 下午2:35
-     */
-    @RequestMapping("/error")
-    @ResponseBody
-    public String handleError(Model model, HttpServletRequest request) {
-        Integer statusCode = (Integer) request.getAttribute("javax.servlet.error.status_code");
-        if (statusCode == 404) {
-            return "接口不存在";
-        } else if (statusCode == 400) {
-            return "参数错误";
-        } else if (statusCode == 500) {
-            return "服务器内部错误";
-        }
-        log.error("http status code: " + statusCode);
+        model.addAttribute("jvmInfo", jvmInfo);
+        model.addAttribute("jvmStat", jvmStat);
+        return "/sys/jvmstatus";
+    }
 
-        model.addAttribute("statusCode", statusCode);
-        model.addAttribute("msg", "页面去火星啦～");
-        return "/404";
+    @ApiOperation(value = "manager 日志页面")
+    @GetMapping("/sys/managerlog/{machineId}")
+    public String managerlogPage(@PathVariable("machineId") String machineId, Model model) {
+        return "/sys/jvmstatus";
     }
 }

manager/src/main/java/cn/reghao/devops/manager/home/controller/page/LoginPageController.java

@@ -0,0 +1,56 @@
+package cn.reghao.devops.manager.home.controller.page;
+
+import cn.reghao.devops.manager.home.service.HomeService;
+import io.swagger.annotations.Api;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.web.servlet.error.ErrorController;
+import org.springframework.stereotype.Controller;
+import org.springframework.ui.Model;
+import org.springframework.web.bind.annotation.*;
+
+import javax.servlet.http.HttpServletRequest;
+
+/**
+ * @author reghao
+ * @date 2021-04-04 21:24:18
+ */
+@Slf4j
+@Api(tags = "登录页和首页")
+@Controller
+public class LoginPageController implements ErrorController {
+    @GetMapping("/login")
+    public String toLogin(Model model) {
+        model.addAttribute("isCaptcha", false);
+        return "/login";
+    }
+
+    @Override
+    public String getErrorPath() {
+        return "/error";
+    }
+
+    /**
+     * 处理错误页面
+     *
+     * @param
+     * @return
+     * @date 2021-05-19 下午2:35
+     */
+    @RequestMapping("/error")
+    @ResponseBody
+    public String handleError(Model model, HttpServletRequest request) {
+        Integer statusCode = (Integer) request.getAttribute("javax.servlet.error.status_code");
+        if (statusCode == 404) {
+            return "接口不存在";
+        } else if (statusCode == 400) {
+            return "参数错误";
+        } else if (statusCode == 500) {
+            return "服务器内部错误";
+        }
+        log.error("http status code: " + statusCode);
+
+        model.addAttribute("statusCode", statusCode);
+        model.addAttribute("msg", "页面去火星啦～");
+        return "/404";
+    }
+}

manager/src/main/java/cn/reghao/devops/manager/home/controller/page/SysPageController.java

@@ -1,44 +0,0 @@
-package cn.reghao.devops.manager.home.controller.page;
-
-import cn.reghao.devops.common.util.jvm.JVM;
-import cn.reghao.devops.common.util.jvm.po.JvmInfo;
-import cn.reghao.devops.common.util.jvm.po.JvmStat;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import org.springframework.stereotype.Controller;
-import org.springframework.ui.Model;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestMapping;
-
-/**
- * @author reghao
- * @date 2021-04-04 21:24:18
- */
-@Api(tags = "系统页面")
-@RequestMapping("/sys")
-@Controller
-public class SysPageController {
-    private final JVM jvm;
-
-    public SysPageController() {
-        this.jvm = new JVM();
-    }
-
-    @ApiOperation(value = "JVM 状态页面")
-    @GetMapping("/jvmstatus")
-    public String jvmStatusPage(Model model) {
-        JvmInfo jvmInfo = jvm.info();
-        JvmStat jvmStat = jvm.stat();
-
-        model.addAttribute("jvmInfo", jvmInfo);
-        model.addAttribute("jvmStat", jvmStat);
-        return "/sys/jvmstatus";
-    }
-
-    @ApiOperation(value = "manager 日志页面")
-    @GetMapping("/managerlog/{machineId}")
-    public String managerlogPage(@PathVariable("machineId") String machineId, Model model) {
-        return "/sys/jvmstatus";
-    }
-}

manager/src/main/java/cn/reghao/devops/manager/ws/handler/LogHandler.java

@@ -1,17 +1,24 @@
 package cn.reghao.devops.manager.ws.handler;
 
+import cn.reghao.devops.model.NginxLog;
 import cn.reghao.devops.manager.log.service.LogService;
 import cn.reghao.jutil.jdk.converter.DateTimeConverter;
 import cn.reghao.jutil.jdk.result.AppLog;
 import cn.reghao.jutil.jdk.serializer.JdkSerializer;
 import cn.reghao.jutil.jdk.serializer.JsonConverter;
+import cn.reghao.jutil.jdk.thread.ThreadPoolWrapper;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
 import org.springframework.web.socket.*;
 
 import java.io.IOException;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
 
 /**
  * @author reghao
@@ -20,11 +27,13 @@ import java.util.concurrent.ConcurrentHashMap;
 @Slf4j
 @Component
 public class LogHandler implements WebSocketHandler {
-    private LogService logService;
+    private final ExecutorService threadPool = ThreadPoolWrapper.threadPool("push-task", 1);
+    private final LogService logService;
     private final Map<String, WebSocketSession> pullSessions = new ConcurrentHashMap<>();
 
     public LogHandler(LogService logService) {
         this.logService = logService;
+        threadPool.submit(new PushTask());
     }
 
     @Override
@@ -37,7 +46,7 @@ public class LogHandler implements WebSocketHandler {
             String host = map.get("host");
             String key = String.format("%s-%s", app, host);
             //pushSessions.put(key, webSocketSession);
-        } else {
+        } else if (path.equals("/ws/log/pull")) {
             String query = webSocketSession.getUri().getQuery();
             Map<String, String> map = parseParams(query);
             String app = map.get("app");
@@ -60,6 +69,7 @@ public class LogHandler implements WebSocketHandler {
         return map;
     }
 
+    Map<Long, Integer> map = new TreeMap<>();
     @Override
     public void handleMessage(WebSocketSession webSocketSession, WebSocketMessage<?> webSocketMessage)
             throws IOException {
@@ -87,6 +97,21 @@ public class LogHandler implements WebSocketHandler {
                             e.printStackTrace();
                         }
                     }
+                } else if (object instanceof NginxLog) {
+                    NginxLog nginxLog = (NginxLog) object;
+                    String date = nginxLog.getTimeIso8601();
+                    DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss+08:00")
+                            .withZone(ZoneId.of("UTC"));
+                    LocalDateTime localDateTime = LocalDateTime.parse(date, formatter);
+                    Long timestamp = localDateTime.toEpochSecond(ZoneOffset.of("+8"));
+                    Long key = timestamp;
+                    Integer count = map.get(key);
+                    if (count == null) {
+                        map.put(key, 1);
+                    } else {
+                        int count1 = map.get(key) + 1;
+                        map.put(key, count1);
+                    }
                 }
             } else if (webSocketMessage instanceof PingMessage) {
                 log.info("接收到 WebSocket PingMessage");
@@ -145,4 +170,45 @@ public class LogHandler implements WebSocketHandler {
     public boolean supportsPartialMessages() {
         return false;
     }
+
+    class PushTask implements Runnable {
+        @Override
+        public void run() {
+            while (!Thread.interrupted()) {
+                try {
+                    if (map.size() < 3) {
+                        Thread.sleep(10_000);
+                        continue;
+                    }
+
+                    LocalDateTime localDateTime = LocalDateTime.now();
+                    Long baseKey = localDateTime.toEpochSecond(ZoneOffset.of("+8"));
+
+                    List<String> xList = new ArrayList<>();
+                    List<Integer> yList = new ArrayList<>();
+                    Set<Long> keys = new HashSet<>();
+                    for (Long key : map.keySet()) {
+                        if (key < baseKey) {
+                            xList.add(DateTimeConverter.format(key*1000).split(" ")[1]);
+                            yList.add(map.get(key));
+                            keys.add(key);
+                        }
+                    }
+
+                    keys.forEach(map::remove);
+                    keys.clear();
+                    List results = new ArrayList();
+                    results.add(xList.toArray());
+                    results.add(yList.toArray());
+
+                    WebSocketSession webSocketSession = getPullSession("admin-service", "172.16.90.200");
+                    TextMessage textMessage = new TextMessage(JsonConverter.objectToJson(results));
+                    webSocketSession.sendMessage(textMessage);
+                    Thread.sleep(10_000);
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+    }
 }

manager/src/test/java/BuildTest.java

@@ -3,11 +3,20 @@ import cn.reghao.devops.common.build.tool.repo.GitImpl;
 import cn.reghao.devops.common.docker.DockerImpl;
 import cn.reghao.devops.common.ws.WsClient;
 import cn.reghao.devops.manager.app.model.po.config.build.RepoAuthConfig;
+import cn.reghao.devops.manager.app.model.vo.ChartData;
+import cn.reghao.devops.model.NginxLog;
+import cn.reghao.jutil.jdk.converter.DateTimeConverter;
+import cn.reghao.jutil.jdk.serializer.JsonConverter;
+import cn.reghao.jutil.jdk.text.TextFile;
 import com.github.dockerjava.api.command.InspectContainerResponse;
 import com.github.dockerjava.api.model.Container;
 import org.junit.Test;
 
-import java.util.List;
+import java.text.ParseException;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.util.*;
 import java.util.regex.Pattern;
 
 /**
@@ -66,4 +75,101 @@ public class BuildTest {
 
         Thread.sleep(3600_000);
     }
+
+    @Test
+    public void test() throws ParseException {
+        Map<String, Integer> map = new TreeMap<>();
+
+        TextFile textFile = new TextFile();
+        String filePath = "/home/reghao/Downloads/api.iquizoo.com.access.log";
+        List<String> list = textFile.read(filePath);
+        for (String line : list) {
+            String address = line.split(" - ")[0];
+            String date = line.split("- \\[")[1].split("]")[0];
+            LocalDateTime localDateTime = DateTimeConverter.localDateTime1(date);
+            String dateTimeStr = DateTimeConverter.format(localDateTime);
+
+            String key = dateTimeStr;
+            Integer count = map.get(key);
+            if (count == null) {
+                map.put(key, 1);
+            } else {
+                int count1 = map.get(key) + 1;
+                map.put(key, count1);
+            }
+        }
+
+        List<Map.Entry<String, Integer>> list1 = new ArrayList<Map.Entry<String, Integer>>(map.entrySet());
+        //然后通过比较器来实现排序
+        Collections.sort(list1, new Comparator<Map.Entry<String, Integer>>() {
+            //升序排序
+            public int compare(Map.Entry<String, Integer> o1, Map.Entry<String, Integer> o2) {
+                return o2.getKey().compareTo(o1.getKey());
+            }
+        });
+
+        for(Map.Entry<String, Integer> mapping : list1) {
+            String key = mapping.getKey();
+            int value = mapping.getValue();
+
+            ChartData chartData = new ChartData();
+            //chartData.setTotal(value);
+            Object obj1 = key;
+            Object obj2 = Integer.valueOf(-1);
+            List<Object> list2 = new ArrayList<>();
+            list2.add(obj1);
+            list2.add(obj2);
+            //chartData.setTime(List.of(list2));
+            System.out.println(mapping.getKey() + " -> " +mapping.getValue());
+        }
+    }
+
+    @Test
+    public void logTest() throws ParseException {
+        /*File file = new File("");
+        JsonConverter.jsonFileToObject(file, NginxLog.class);*/
+
+        TextFile textFile = new TextFile();
+        String filePath = "/home/reghao/Downloads/api.iquizoo.com.access1.log";
+        filePath = "/home/reghao/Downloads/api.iquizoo.com.access.log-20231107";
+        filePath = "/home/reghao/Downloads/api.iquizoo.com.access.log-20231108";
+
+        List<String> list = textFile.read(filePath);
+        List<NginxLog> nginxLogs = new ArrayList<>();
+        for (String line : list) {
+            if (!line.startsWith("{")) {
+                continue;
+            }
+
+            try {
+                NginxLog nginxLog = JsonConverter.jsonToObject(line, NginxLog.class);
+                nginxLogs.add(nginxLog);
+            } catch (Exception e) {
+                // e.printStackTrace();
+            }
+        }
+
+        Map<String, Integer> map = new TreeMap<>();
+        for (NginxLog nginxLog : nginxLogs) {
+            String date = nginxLog.getTimeIso8601();
+            DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss+08:00")
+                    .withZone(ZoneId.of("UTC"));
+            LocalDateTime localDateTime = LocalDateTime.parse(date, formatter);
+            String dateTimeStr = DateTimeConverter.format(localDateTime);
+
+            String key = dateTimeStr;
+            Integer count = map.get(key);
+            if (count == null) {
+                map.put(key, 1);
+            } else {
+                int count1 = map.get(key) + 1;
+                map.put(key, count1);
+            }
+        }
+
+        System.out.println();
+        map.forEach((key, value) -> {
+            System.out.println(key + " -> " + value);
+        });
+    }
 }