update search-service test case

search/search-service/src/test/java/ElasticTest.java

@@ -1,66 +1,24 @@
-import ch.qos.logback.classic.Level;
-import ch.qos.logback.classic.Logger;
-import ch.qos.logback.classic.LoggerContext;
-import cn.reghao.jutil.jdk.string.SnowFlake;
 import cn.reghao.tnb.content.api.constant.PostScope;
-import cn.reghao.tnb.search.app.SearchApplication;
+import cn.reghao.tnb.search.app.config.ElasticProperties;
 import cn.reghao.tnb.search.app.es.*;
 import cn.reghao.tnb.search.app.model.po.VideoText;
-import co.elastic.clients.elasticsearch._types.mapping.Property;
-import co.elastic.clients.elasticsearch.indices.AnalyzeRequest;
-import lombok.extern.slf4j.Slf4j;
 import org.junit.jupiter.api.Test;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
-import org.springframework.test.context.ActiveProfiles;
 
-import java.io.IOException;
 import java.util.*;
 
 /**
  * @author reghao
  * @date 2025-03-18 15:51:21
  */
-@Slf4j
-@ActiveProfiles("dev")
-@SpringBootTest(classes = SearchApplication.class)
 public class ElasticTest {
-    /*void setLogLevel() {
-        LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
-        Logger rootLogger = loggerContext.getLogger("ROOT");
-        rootLogger.setLevel(Level.INFO);
-    }
-
-    SnowFlake idGenerator = new SnowFlake(1, 1);
-    @Autowired
-    ElasticService elasticService;
-    @Autowired
-    IndexService indexService;
-    @Autowired
-    MappingService mappingService;
-    @Autowired
-    DocumentService documentService;
-
-    @Test
-    public void indexTest() throws IOException {
-        String indexName = "video_text";
-
-        indexService.deleteIndex(indexName);
-        Map<String, Property> propertyMap = mappingService.getVideoTextPropertyMap();
-        indexService.createIndex(indexName, propertyMap);
-    }
-
-    @Autowired
     VideoTextDocument videoTextDocument;
     @Test
     public void documentTest() {
         videoTextDocument.deleteAllDocument();
     }
 
-    @Autowired
     SearchService searchService;
     @Test
     public void storeTest() throws Exception {
@@ -71,23 +29,10 @@ public class ElasticTest {
 
         String index = VideoText.class.getSimpleName().toLowerCase(Locale.ROOT);
         String kw = "隔壁";
-        *//*Page<VideoSummary> page = hibernateQuery.queryWithHighlight(kw, pn, ps);
-        Page<VideoSummary> page1 = luceneQuery.queryWithHighlight("", kw, pn, ps);*//*
         //Page<VideoSummary> page2 = esQuery.queryWithHighlight(index, kw, pn, ps, VideoSummary.class);
         System.out.println();
-
-        //Map<String, Property> propertyMap = mappingService.getVideoTextPropertyMap();
-//        indexService.deleteIndex(index);
-//        indexService.createIndex(index, propertyMap);
-        //indexService.getIndex(index);
-        //indexService.deleteIndex(index);
-        //indexService.getIndex(index);
-        //indexService.updateMapping(index);
-        //searchService.searchAll(index);
-        //searchService.count(index);
     }
 
-    @Autowired
     VideoTextQuery videoTextQuery;
     @Test
     public void searchTest() {
@@ -103,17 +48,31 @@ public class ElasticTest {
     }
 
     @Test
-    public void analyzerTest() throws IOException {
-        String text = "中华人民共和国国歌";
-        String analyzer = "standard";
-        analyzer = "ik_max_word";
-        //analyzer = "ik_smart";
-//        AnalyzeRequest analyzeRequest = new AnalyzeRequest.Builder()
-//                .text(text)
-//                .analyzer(analyzer)
-//                .build();
-        *//*ElasticsearchClient esClient = elasticService.getElasticsearchClient();
-        AnalyzeResponse analyzeResponse = esClient.indices().analyze(analyzeRequest);
-        List<AnalyzeToken> tokens = analyzeResponse.tokens();*//*
-    }*/
+    public void logTest() throws Exception {
+        ElasticProperties elasticProperties = new ElasticProperties();
+        elasticProperties.setHost("192.168.0.81");
+        elasticProperties.setPort(9200);
+        elasticProperties.setUsername("elastic");
+        elasticProperties.setPassword("VLTtN03SSJ4lsyyg56kf");
+
+        ElasticService elasticService = new ElasticService(elasticProperties);
+        IndexService indexService = new IndexService(elasticService);
+        MappingService mappingService = new MappingService();
+        SearchService searchService = new SearchService(elasticService);
+
+        String indexName = "nginx_log";
+        String aggregateField = "timeIso8601";
+        //Map<String, Long> dateGroupMap = searchService.aggregateByDay(indexName, aggregateField);
+
+        aggregateField = "url.raw";
+        String dateField = "timeIso8601";
+        String dateValue = "2023-11-09";
+        //Map<String, Long> urlGroupMap = searchService.aggregateByUrl(indexName, aggregateField, dateField, dateValue);
+
+        long total = searchService.count(indexName);
+        System.out.printf("Total documents of %s: %d%n", indexName, total);
+
+        DocumentService documentService = new DocumentService(elasticService);
+        documentService.deleteAllDocument(indexName);
+    }
 }

search/search-service/src/test/java/NginxLogTest.java

@@ -1,10 +1,16 @@
 import cn.reghao.jutil.jdk.serializer.JsonConverter;
 import cn.reghao.jutil.jdk.string.SnowFlake;
 import cn.reghao.tnb.search.app.SearchApplication;
-import cn.reghao.tnb.search.app.es.DocumentService;
-import cn.reghao.tnb.search.app.es.QueryService;
-import cn.reghao.tnb.search.app.es.SearchService;
-import cn.reghao.tnb.search.app.log.model.NginxLog;
+import cn.reghao.tnb.search.app.config.ElasticProperties;
+import cn.reghao.tnb.search.app.es.*;
+import cn.reghao.jutil.jdk.web.log.NginxLog;
+import cn.reghao.tnb.search.app.log.NginxLogDocument;
+import cn.reghao.tnb.search.app.log.NginxLogSearch;
+import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.elasticsearch._types.mapping.Property;
+import co.elastic.clients.elasticsearch.indices.AnalyzeRequest;
+import co.elastic.clients.elasticsearch.indices.AnalyzeResponse;
+import co.elastic.clients.elasticsearch.indices.analyze.AnalyzeToken;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -18,6 +24,7 @@ import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 /**
  * @author reghao
@@ -27,12 +34,36 @@ import java.util.List;
 @ActiveProfiles("dev")
 @SpringBootTest(classes = SearchApplication.class)
 public class NginxLogTest {
-    /*@Autowired
+    @Autowired
     QueryService<NginxLog> queryService;
     @Autowired
     SearchService searchService;
     @Autowired
     DocumentService documentService;
+    @Autowired
+    IndexService indexService;
+    @Autowired
+    MappingService mappingService;
+    @Autowired
+    ElasticService elasticService;
+
+    @Test
+    public void indexTest() throws IOException {
+        String indexName = "nginx_log";
+        indexService.deleteIndex(indexName);
+        Map<String, Property> propertyMap = mappingService.getPropertyMapWithNginxLog(NginxLog.class);
+        indexService.createIndex(indexName, propertyMap);
+
+        //Map<String, Property> propertyMap = mappingService.getVideoTextPropertyMap();
+//        indexService.deleteIndex(index);
+//        indexService.createIndex(index, propertyMap);
+        //indexService.getIndex(index);
+        //indexService.deleteIndex(indexName);
+        //indexService.getIndex(index);
+        //indexService.updateMapping(index);
+        //searchService.searchAll(index);
+        //searchService.count(index);
+    }
 
     static SnowFlake idGenerator = new SnowFlake(1, 1);
     static long total = 0L;
@@ -72,12 +103,12 @@ public class NginxLogTest {
                 }
 
                 while (nginxLogs.size() > 10_000) {
-                    //nginxLogs.forEach(nginxLog -> nginxLog.setId(idGenerator.nextId()+""));
+                    nginxLogs.forEach(nginxLog -> nginxLog.setId(idGenerator.nextId()+""));
                     //NginxLog nginxLog = nginxLogs.get(0);
                     //documentService.update(index, nginxLog);
 
-                    //documentService.batchAddDocument(index, nginxLogs);
-                    //log.info("save {} nginxLogs", nginxLogs.size());
+                    documentService.batchAddDocument(index, nginxLogs);
+                    log.info("save {} nginxLogs", nginxLogs.size());
                     total += nginxLogs.size();
                     nginxLogs.clear();
                 }
@@ -89,7 +120,9 @@ public class NginxLogTest {
         }
     }
 
-    void addNginxLogs(String index) {
+    @Test
+    public void addNginxLogTest() {
+        String index = "nginx_log";
         String baseDir = "/home/reghao/work/azy/data/konglog";
         File dir = new File(baseDir);
         for (File file : dir.listFiles()) {
@@ -100,20 +133,73 @@ public class NginxLogTest {
     }
 
     @Test
-    public void addNginxLogTest() {
+    public void nginxLogTest() throws Exception {
         String index = "nginx_log";
-        addNginxLogs(index);
+        int pageSize = 1000;
+        int pageNumber = 1;
+        String fieldName = "url.raw";
+        String fieldValue = "/datareceive/ReceiveData/SendContentResult";
+        //List<NginxLog> list = searchService.searchByPage(index, pageSize, pageNumber, fieldName, fieldValue);
+
+        String aggregateField = "url.raw";
+        aggregateField = "timeIso8601";
+        searchService.aggregate(index, aggregateField);
+
+        //ElasticProperties elasticProperties = new ElasticProperties();
+        //ElasticService elasticService = new ElasticService(elasticProperties);
+        //IndexService indexService = new IndexService(elasticService);
+        //MappingService mappingService = new MappingService();
+
+        //NginxLogDocument nginxLogDocument = new NginxLogDocument(elasticService);
+        //NginxLogSearch nginxLogSearch = new NginxLogSearch(elasticService);
+        //QueryService<NginxLog> queryService = new QueryService<>(elasticService);
+        //indexService.getIndex(index);
+        //indexService.getMapping(index);
+
+        //documentService.deleteAllDocument(index);
+        //Map<String, Property> propertyMap = mappingService.getPropertyMapWithNginxLog(NginxLog.class);
+        //indexService.deleteIndex(index);
+        //indexService.createIndex(index, propertyMap);
+
+        //searchService.search(index);
+//        search1(esClient, "app_log");
+//        index = "app_log";
+//        deleteAll(index);
+//        searchService.aggregate(index);
+
+        /*int pn = 1;
+        while (pn < 100) {
+            List<NginxLog> list = searchService.searchByPage(index, pn, "", "");
+            System.out.println();
+            pn++;
+        }*/
+
+        //searchService.searchAll(index);
+//        indexService.updateMapping(index);
+
+        //String queryString = "content";
+        //List<NginxLog> list = queryService.queryWithHighlight(index, queryString, pn, ps, NginxLog.class);
+        //searchService.searchAll(index);
+        //nginxLogSearch.aggregate();
+        //searchService.count(index);
     }
 
     @Test
-    public void nginxLogTest() throws Exception {
-        String index = "nginx_log";
-        int ps = 1000;
-        int pn = 1;
-        String queryString = "content";
-        //Page<NginxLog> list = queryService.queryWithHighlight(index, queryString, pn, ps, NginxLog.class);
-        //searchService.searchAll(index);
-        searchService.aggregate(index);
+    public void analyzerTest() throws IOException {
+        // es 默认的标准分词器
+        String analyzer = "standard";
+        // ik 的两种分词器
+        //analyzer = "ik_max_word";
+        analyzer = "ik_smart";
+
+        String text = "中华人民共和国国歌";
+        AnalyzeRequest analyzeRequest = new AnalyzeRequest.Builder()
+                .text(text)
+                .analyzer(analyzer)
+                .build();
+        ElasticsearchClient esClient = elasticService.getElasticsearchClient();
+        AnalyzeResponse analyzeResponse = esClient.indices().analyze(analyzeRequest);
+        List<AnalyzeToken> tokens = analyzeResponse.tokens();
         System.out.println();
-    }*/
+    }
 }