update search-service test case

search/search-service/src/test/java/ElasticTest.java

@@ -1,78 +1,253 @@
-import cn.reghao.tnb.content.api.constant.PostScope;
-import cn.reghao.tnb.search.app.config.ElasticProperties;
+import cn.reghao.jutil.jdk.serializer.JsonConverter;
+import cn.reghao.jutil.jdk.web.log.NginxLog;
+import cn.reghao.tnb.search.app.config.AppProperties;
 import cn.reghao.tnb.search.app.es.*;
-import cn.reghao.tnb.search.app.model.po.VideoText;
+import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.elasticsearch._types.SortOrder;
+import co.elastic.clients.elasticsearch._types.mapping.Property;
+import co.elastic.clients.elasticsearch._types.query_dsl.Query;
+import co.elastic.clients.elasticsearch._types.query_dsl.RangeQuery;
+import co.elastic.clients.elasticsearch.indices.AnalyzeRequest;
+import co.elastic.clients.elasticsearch.indices.AnalyzeResponse;
+import co.elastic.clients.elasticsearch.indices.analyze.AnalyzeToken;
+import co.elastic.clients.json.JsonData;
+import lombok.extern.slf4j.Slf4j;
 import org.junit.jupiter.api.Test;
-import org.springframework.data.domain.Page;
-import org.springframework.data.domain.PageRequest;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
 import java.util.*;
 
 /**
  * @author reghao
  * @date 2025-03-18 15:51:21
  */
+@Slf4j
 public class ElasticTest {
-    VideoTextDocument videoTextDocument;
-    @Test
-    public void documentTest() {
-        videoTextDocument.deleteAllDocument();
+    ElasticService getElasticService() {
+        AppProperties appProperties = new AppProperties();
+        appProperties.setEsHost("192.168.0.81");
+        appProperties.setEsPort(9200);
+        appProperties.setEsUsername("elastic");
+        appProperties.setEsPassword("VLTtN03SSJ4lsyyg56kf");
+
+        ElasticService elasticService = new ElasticService(appProperties);
+        return elasticService;
     }
 
-    SearchService searchService;
     @Test
-    public void storeTest() throws Exception {
-        int pn = 1;
-        int ps = 12;
-        PageRequest pageRequest = PageRequest.of(pn-1, ps);
-        String videoId = "ao1n8ggYOg";
-
-        String index = VideoText.class.getSimpleName().toLowerCase(Locale.ROOT);
-        String kw = "隔壁";
-        //Page<VideoSummary> page2 = esQuery.queryWithHighlight(index, kw, pn, ps, VideoSummary.class);
+    public void logTest() throws Exception {
+        ElasticService elasticService = getElasticService();
+        SearchService searchService = new SearchService(elasticService);
+
+        String indexName = "nginx_log";
+        int pageSize = 1000;
+        int pageNumber = 1;
+
+        String fieldName = "url.raw";
+        String fieldValue = "/datareceive/ReceiveData/SendContentResult";
+        String sortField = "timeIso8601";
+        sortField = "requestTime";
+        SortOrder sortOrder = SortOrder.Desc;
+
+        String startDate = "2024-05-20";
+        String endDate = "2024-05-20";
+        String start1 = String.format("%sT00:00:00", startDate);
+        String end1 = String.format("%sT23:59:59", endDate);
+        Query dateQuery = RangeQuery.of(q -> q.field("timeIso8601")
+                .gte(JsonData.of(start1)).lte(JsonData.of(end1)))._toQuery();
+
+        Query combinedQuery = Query.of(q -> q.matchAll(m -> m));
+        if (!fieldValue.isBlank()) {
+            Query termQuery = EsQuery.getTermQuery(fieldName, fieldValue);
+            Query termQuery1 = EsQuery.getTermQuery("status", "200");
+            //combinedQuery = Query.of(q -> q.bool(b -> b.filter(termQuery1).filter(termQuery)));
+            combinedQuery = Query.of(q -> q.bool(b -> b.filter(termQuery1).filter(dateQuery)));
+        }
+
+        List<NginxLog> list0 = searchService.searchByPage(indexName, pageSize, pageNumber, combinedQuery, sortField, sortOrder);
         System.out.println();
     }
 
-    VideoTextQuery videoTextQuery;
     @Test
-    public void searchTest() {
-        int pageNumber = 1;
-        String keyword = "隔壁";
+    public void aggregateTest() throws Exception {
+        ElasticService elasticService = getElasticService();
+        SearchService searchService = new SearchService(elasticService);
+
+        String indexName = "nginx_log";
+        String dateField = "timeIso8601";
+        String startDateTime = "2024-08-10 00:00:00";
+        String endDateTime = "2024-08-10 23:59:59";
+
+        String start = startDateTime.replace(" ", "T");
+        String end = endDateTime.replace(" ", "T");
+        RangeQuery dateQuery = RangeQuery.of(q -> q.field(dateField).gte(JsonData.of(start)).lte(JsonData.of(end)));
+        Query query = dateQuery._toQuery();
+
+        String aggregateField = "httpUserAgent.raw";
+        Map<String, Long> groupMap2 = searchService.aggregateByQuery(indexName, aggregateField, query);
 
-        List<Integer> scopes = List.of(PostScope.PUBLIC.getCode(), PostScope.PROTECT.getCode());
-        Page<VideoText> page = videoTextQuery.queryWithHighlight(keyword, scopes, pageNumber);
-        long total = page.getTotalElements();
-        List<VideoText> list = page.getContent();
-        List<VideoText> list1 = videoTextQuery.queryByPage(pageNumber);
+        String windows = "Windows NT";
+        String linux = "Linux ";
+        String mac = "Macintosh";
+        String ipad = "iPad";
+        String iphone = "iPhone";
+        String android = "Android";
+        String client = "client";
+        String others = "others";
+        Map<String, Integer> mapCount = new HashMap<>();
+        mapCount.put(windows, 0);
+        mapCount.put(linux, 0);
+        mapCount.put(mac, 0);
+        mapCount.put(ipad, 0);
+        mapCount.put(iphone, 0);
+        mapCount.put(android, 0);
+        mapCount.put(client, 0);
+        mapCount.put(others, 0);
+
+        groupMap2.keySet().forEach(userAgent -> {
+            if (userAgent.contains(windows)) {
+                incr(windows, mapCount);
+            } else if (userAgent.contains(linux)) {
+                incr(linux, mapCount);
+            } else if (userAgent.contains(mac)) {
+                incr(mac, mapCount);
+            } else if (userAgent.contains(ipad)) {
+                incr(ipad, mapCount);
+            } else if (userAgent.contains(iphone)) {
+                incr(iphone, mapCount);
+            } else if (userAgent.contains(android)) {
+                incr(android, mapCount);
+            } else if (userAgent.contains(client) || userAgent.contains("Client")) {
+                incr(client, mapCount);
+            } else {
+                incr(others, mapCount);
+            }
+        });
         System.out.println();
     }
 
+    private void incr(String key, Map<String, Integer> map) {
+        map.put(key, map.get(key)+1);
+    }
+
     @Test
-    public void logTest() throws Exception {
-        ElasticProperties elasticProperties = new ElasticProperties();
-        elasticProperties.setHost("192.168.0.81");
-        elasticProperties.setPort(9200);
-        elasticProperties.setUsername("elastic");
-        elasticProperties.setPassword("VLTtN03SSJ4lsyyg56kf");
+    public void esDocTest() throws Exception {
+        String indexName = "nginx_log";
+        ElasticService elasticService = getElasticService();
+        SearchService searchService = new SearchService(elasticService);
 
-        ElasticService elasticService = new ElasticService(elasticProperties);
+        Query combinedQuery = Query.of(q -> q.matchAll(m -> m));
+        long total = searchService.count(indexName, combinedQuery);
+        log.info("Total documents of {}: {}", indexName, total);
+
+        DocumentService documentService = new DocumentService(elasticService);
+        //documentService.deleteAllDocument(indexName);
+    }
+
+    @Test
+    public void esIndexTest() throws IOException {
+        String indexName = "nginx_log";
+        ElasticService elasticService = getElasticService();
         IndexService indexService = new IndexService(elasticService);
         MappingService mappingService = new MappingService();
-        SearchService searchService = new SearchService(elasticService);
 
-        String indexName = "nginx_log";
-        String aggregateField = "timeIso8601";
-        //Map<String, Long> dateGroupMap = searchService.aggregateByDay(indexName, aggregateField);
+        indexService.deleteIndex(indexName);
+        Map<String, Property> propertyMap = mappingService.getPropertyMapWithNginxLog(NginxLog.class);
+        indexService.createIndex(indexName, propertyMap);
+    }
 
-        aggregateField = "url.raw";
-        String dateField = "timeIso8601";
-        String dateValue = "2023-11-09";
-        //Map<String, Long> urlGroupMap = searchService.aggregateByUrl(indexName, aggregateField, dateField, dateValue);
+    @Test
+    public void nginxLogTest() {
+        ElasticService elasticService = getElasticService();
+        DocumentService documentService = new DocumentService(elasticService);
 
-        long total = searchService.count(indexName);
-        System.out.printf("Total documents of %s: %d%n", indexName, total);
+        String filePath = "/home/reghao/Downloads/access-20231107_073356-20240905_165944.log";
+        String indexName = "nginx_log1";
+        readFileFileChannel(new File(filePath), indexName, documentService);
+    }
 
-        DocumentService documentService = new DocumentService(elasticService);
-        documentService.deleteAllDocument(indexName);
+    void readFileFileChannel(File file, String index, DocumentService documentService) {
+        List<String> lines = new ArrayList<>();
+        List<NginxLog> nginxLogs = new ArrayList<>();
+        try {
+            FileInputStream fis = new FileInputStream(file);
+            FileChannel fileChannel = fis.getChannel();
+
+            // 10MB
+            int capacity = 10*1024*1024;
+            ByteBuffer byteBuffer = ByteBuffer.allocate(capacity);
+            StringBuffer buffer = new StringBuffer();
+            while(fileChannel.read(byteBuffer) != -1) {
+                //读取后，将位置置为0，将limit置为容量, 以备下次读入到字节缓冲中，从0开始存储
+                byteBuffer.clear();
+                byte[] bytes = byteBuffer.array();
+
+                String str = new String(bytes);
+                buffer.append(str);
+                String[] strArray = buffer.toString().split(System.lineSeparator());
+                for (int i = 0; i < strArray.length-1; i++) {
+                    try {
+                        NginxLog nginxLog = JsonConverter.jsonToObject(strArray[i], NginxLog.class);
+                        String timeStr = nginxLog.getTimeIso8601().replace("+08:00", "");
+                        nginxLog.setTimeIso8601(timeStr);
+                        //LocalDateTime localDateTime = DateTimeConverter.localDateTime(nginxLog.getTimeIso8601());
+                        //long timestamp = DateTimeConverter.msTimestamp(localDateTime);
+                        //String localDateTimeStr = DateTimeConverter.format(localDateTime);
+                        //nginxLog.setRequestTimestamp(timestamp);
+                        try {
+                            documentService.addDocument(index, nginxLog);
+                        } catch (IOException e) {
+                            e.printStackTrace();
+                        }
+
+                        nginxLogs.add(nginxLog);
+                        if (nginxLogs.size() > 10) {
+                            //nginxLogs.forEach(nginxLog -> nginxLog.setId(idGenerator.nextId()+""));
+                            //NginxLog nginxLog = nginxLogs.get(0);
+                            //documentService.update(index, nginxLog);
+
+                            //documentService.batchAddDocument(index, nginxLogs);
+                            //log.info("save {} nginxLogs", nginxLogs.size());
+                            break;
+                        }
+                    } catch (Exception e) {
+                        lines.add(strArray[i]);
+                        //e.printStackTrace();
+                    }
+                }
+
+                if (nginxLogs.size() > 10) {
+                    break;
+                }
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        } finally {
+            // TODO close 处理
+        }
+    }
+
+    @Test
+    public void analyzerTest() throws IOException {
+        ElasticService elasticService = getElasticService();
+        // es 默认的标准分词器
+        String analyzer = "standard";
+        // ik 的两种分词器
+        //analyzer = "ik_max_word";
+        analyzer = "ik_smart";
+
+        String text = "中华人民共和国国歌";
+        AnalyzeRequest analyzeRequest = new AnalyzeRequest.Builder()
+                .text(text)
+                .analyzer(analyzer)
+                .build();
+        ElasticsearchClient esClient = elasticService.getElasticsearchClient();
+        AnalyzeResponse analyzeResponse = esClient.indices().analyze(analyzeRequest);
+        List<AnalyzeToken> tokens = analyzeResponse.tokens();
+        System.out.println();
     }
 }

search/search-service/src/test/java/NginxLogTest.java

@@ -1,16 +1,9 @@
 import cn.reghao.jutil.jdk.serializer.JsonConverter;
 import cn.reghao.jutil.jdk.string.SnowFlake;
 import cn.reghao.tnb.search.app.SearchApplication;
-import cn.reghao.tnb.search.app.config.ElasticProperties;
 import cn.reghao.tnb.search.app.es.*;
 import cn.reghao.jutil.jdk.web.log.NginxLog;
-import cn.reghao.tnb.search.app.log.NginxLogDocument;
-import cn.reghao.tnb.search.app.log.NginxLogSearch;
-import co.elastic.clients.elasticsearch.ElasticsearchClient;
 import co.elastic.clients.elasticsearch._types.mapping.Property;
-import co.elastic.clients.elasticsearch.indices.AnalyzeRequest;
-import co.elastic.clients.elasticsearch.indices.AnalyzeResponse;
-import co.elastic.clients.elasticsearch.indices.analyze.AnalyzeToken;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -62,7 +55,6 @@ public class NginxLogTest {
         //indexService.getIndex(index);
         //indexService.updateMapping(index);
         //searchService.searchAll(index);
-        //searchService.count(index);
     }
 
     static SnowFlake idGenerator = new SnowFlake(1, 1);
@@ -143,9 +135,9 @@ public class NginxLogTest {
 
         String aggregateField = "url.raw";
         aggregateField = "timeIso8601";
-        searchService.aggregate(index, aggregateField);
+        //searchService.aggregate(index, aggregateField);
 
-        //ElasticProperties elasticProperties = new ElasticProperties();
+        //AppProperties elasticProperties = new AppProperties();
         //ElasticService elasticService = new ElasticService(elasticProperties);
         //IndexService indexService = new IndexService(elasticService);
         //MappingService mappingService = new MappingService();
@@ -181,25 +173,5 @@ public class NginxLogTest {
         //List<NginxLog> list = queryService.queryWithHighlight(index, queryString, pn, ps, NginxLog.class);
         //searchService.searchAll(index);
         //nginxLogSearch.aggregate();
-        //searchService.count(index);
-    }
-
-    @Test
-    public void analyzerTest() throws IOException {
-        // es 默认的标准分词器
-        String analyzer = "standard";
-        // ik 的两种分词器
-        //analyzer = "ik_max_word";
-        analyzer = "ik_smart";
-
-        String text = "中华人民共和国国歌";
-        AnalyzeRequest analyzeRequest = new AnalyzeRequest.Builder()
-                .text(text)
-                .analyzer(analyzer)
-                .build();
-        ElasticsearchClient esClient = elasticService.getElasticsearchClient();
-        AnalyzeResponse analyzeResponse = esClient.indices().analyze(analyzeRequest);
-        List<AnalyzeToken> tokens = analyzeResponse.tokens();
-        System.out.println();
     }
 }