search-service 模块添加 NginxLogTest

search/search-service/src/main/java/cn/reghao/tnb/search/app/LoggingService.java → search/search-service/src/main/java/cn/reghao/tnb/search/app/NginxLogService.java

@@ -1,25 +1,15 @@
 package cn.reghao.tnb.search.app;
 
-import ch.qos.logback.classic.Level;
-import ch.qos.logback.classic.Logger;
-import ch.qos.logback.classic.LoggerContext;
 import cn.reghao.tnb.search.app.config.ElasticProperties;
 import cn.reghao.tnb.search.app.es.*;
 import cn.reghao.tnb.search.app.model.po.NginxLog;
 import cn.reghao.jutil.jdk.converter.DateTimeConverter;
 import cn.reghao.jutil.jdk.serializer.JsonConverter;
 import cn.reghao.jutil.jdk.text.TextFile;
-import cn.reghao.jutil.tool.id.SnowFlake;
 import co.elastic.clients.elasticsearch._types.mapping.Property;
 import lombok.extern.slf4j.Slf4j;
-import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
 import java.text.ParseException;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
@@ -33,9 +23,7 @@ import java.util.*;
  */
 @Slf4j
 @Service
-public class LoggingService {
-    static SnowFlake idGenerator = new SnowFlake(1, 1);
-
+public class NginxLogService {
     public List getChartData() throws ParseException {
         TextFile textFile = new TextFile();
         String filePath = "nginx.log";
@@ -96,69 +84,7 @@ public class LoggingService {
         return results;
     }
 
-    public static void readFileFileChannel(String filePath, String index, DocumentService documentService) {
-        List<String> lines = new ArrayList<>();
-        List<NginxLog> nginxLogs = new ArrayList<>();
-        File file = new File(filePath);
-        try {
-            FileInputStream fis = new FileInputStream(file);
-            FileChannel fileChannel = fis.getChannel();
-
-            // 10MB
-            int capacity = 10*1024*1024;
-            ByteBuffer byteBuffer = ByteBuffer.allocate(capacity);
-            StringBuffer buffer = new StringBuffer();
-            while(fileChannel.read(byteBuffer) != -1) {
-                //读取后，将位置置为0，将limit置为容量, 以备下次读入到字节缓冲中，从0开始存储
-                byteBuffer.clear();
-                byte[] bytes = byteBuffer.array();
-
-                String str = new String(bytes);
-                buffer.append(str);
-                String[] strArray = buffer.toString().split(System.lineSeparator());
-                for (int i = 0; i < strArray.length-1; i++) {
-                    try {
-                        NginxLog nginxLog = JsonConverter.jsonToObject(strArray[i], NginxLog.class);
-                        nginxLogs.add(nginxLog);
-                    } catch (Exception e) {
-                        lines.add(strArray[i]);
-                        //e.printStackTrace();
-                    }
-                }
-
-                String lastLine = strArray[strArray.length-1];
-                if (!lastLine.endsWith("}")) {
-                    buffer = new StringBuffer();
-                    buffer.append(strArray[strArray.length-1]);
-                }
-
-                while (nginxLogs.size() > 10_000) {
-                    nginxLogs.forEach(nginxLog -> nginxLog.setId(idGenerator.nextId()+""));
-
-                    //NginxLog nginxLog = nginxLogs.get(0);
-                    //documentService.update(index, nginxLog);
-
-                    documentService.batchAddDocument(index, nginxLogs);
-                    log.info("save {} nginxLogs", nginxLogs.size());
-                    nginxLogs.clear();
-                }
-            }
-        } catch (IOException e) {
-            e.printStackTrace();
-        } finally {
-            // TODO close 处理
-        }
-    }
-
-    static void setLogLevel() {
-        LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
-        Logger rootLogger = loggerContext.getLogger("ROOT");
-        rootLogger.setLevel(Level.INFO);
-    }
-
     public static void main(String[] args) throws Exception {
-        setLogLevel();
-
         ElasticProperties elasticProperties = new ElasticProperties();
         ElasticService elasticService = new ElasticService(elasticProperties);
         IndexService indexService = new IndexService(elasticService);
@@ -171,10 +97,6 @@ public class LoggingService {
         //indexService.getIndex(index);
         //indexService.getMapping(index);
 
-        String filePath = "/home/reghao/work/azy/data/access-20231107_073356-20240905_165944.log";
-        filePath = "/home/reghao/work/azy/data/access-20240905_165948-20250306_170553.log";
-        //readFileFileChannel(filePath, index, documentService);
-
         //documentService.deleteAllDocument(index);
         Map<String, Property> propertyMap = mappingService.getPropertyMap(NginxLog.class);
         //indexService.deleteIndex(index);

search/search-service/src/test/java/NginxLogTest.java

@@ -0,0 +1,120 @@
+import cn.reghao.jutil.jdk.serializer.JsonConverter;
+import cn.reghao.jutil.tool.id.SnowFlake;
+import cn.reghao.tnb.search.app.SearchApplication;
+import cn.reghao.tnb.search.app.es.DocumentService;
+import cn.reghao.tnb.search.app.es.QueryService;
+import cn.reghao.tnb.search.app.es.SearchService;
+import cn.reghao.tnb.search.app.model.po.NginxLog;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.ActiveProfiles;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author reghao
+ * @date 2025-07-03 15:56:02
+ */
+@Slf4j
+@ActiveProfiles("dev")
+@SpringBootTest(classes = SearchApplication.class)
+public class NginxLogTest {
+    @Autowired
+    QueryService<NginxLog> queryService;
+    @Autowired
+    SearchService searchService;
+    @Autowired
+    DocumentService documentService;
+
+    static SnowFlake idGenerator = new SnowFlake(1, 1);
+    static long total = 0L;
+    public static void readFileFileChannel(File file, String index, DocumentService documentService) {
+        List<String> lines = new ArrayList<>();
+        List<NginxLog> nginxLogs = new ArrayList<>();
+        try {
+            FileInputStream fis = new FileInputStream(file);
+            FileChannel fileChannel = fis.getChannel();
+
+            // 10MB
+            int capacity = 10*1024*1024;
+            ByteBuffer byteBuffer = ByteBuffer.allocate(capacity);
+            StringBuffer buffer = new StringBuffer();
+            while(fileChannel.read(byteBuffer) != -1) {
+                //读取后，将位置置为0，将limit置为容量, 以备下次读入到字节缓冲中，从0开始存储
+                byteBuffer.clear();
+                byte[] bytes = byteBuffer.array();
+
+                String str = new String(bytes);
+                buffer.append(str);
+                String[] strArray = buffer.toString().split(System.lineSeparator());
+                for (int i = 0; i < strArray.length-1; i++) {
+                    try {
+                        NginxLog nginxLog = JsonConverter.jsonToObject(strArray[i], NginxLog.class);
+                        nginxLogs.add(nginxLog);
+                    } catch (Exception e) {
+                        lines.add(strArray[i]);
+                        //e.printStackTrace();
+                    }
+                }
+
+                String lastLine = strArray[strArray.length-1];
+                if (!lastLine.endsWith("}")) {
+                    buffer = new StringBuffer();
+                    buffer.append(strArray[strArray.length-1]);
+                }
+
+                while (nginxLogs.size() > 10_000) {
+                    //nginxLogs.forEach(nginxLog -> nginxLog.setId(idGenerator.nextId()+""));
+                    //NginxLog nginxLog = nginxLogs.get(0);
+                    //documentService.update(index, nginxLog);
+
+                    //documentService.batchAddDocument(index, nginxLogs);
+                    //log.info("save {} nginxLogs", nginxLogs.size());
+                    total += nginxLogs.size();
+                    nginxLogs.clear();
+                }
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        } finally {
+            // TODO close 处理
+        }
+    }
+
+    void addNginxLogs(String index) {
+        String baseDir = "/home/reghao/work/azy/data/konglog";
+        File dir = new File(baseDir);
+        for (File file : dir.listFiles()) {
+            readFileFileChannel(file, index, documentService);
+            log.info("total -> {}", total);
+        }
+        log.info("total -> {}", total);
+    }
+
+    @Test
+    public void addNginxLogTest() {
+        String index = "nginx_log";
+        addNginxLogs(index);
+    }
+
+    @Test
+    public void nginxLogTest() throws Exception {
+        String index = "nginx_log";
+        int ps = 1000;
+        int pn = 1;
+        String queryString = "content";
+        //Page<NginxLog> list = queryService.queryWithHighlight(index, queryString, pn, ps, NginxLog.class);
+        //searchService.searchAll(index);
+        searchService.aggregate(index);
+        System.out.println();
+    }
+}