update search-service

search/search-api/src/main/java/cn/reghao/tnb/search/api/iface/DataSearchService.java

@@ -3,13 +3,16 @@ package cn.reghao.tnb.search.api.iface;
 import cn.reghao.tnb.search.api.dto.VideoSummary;
 import cn.reghao.jutil.jdk.db.PageList;
 
+import java.util.List;
+
 /**
  * @author reghao
  * @date 2025-03-27 13:45:23
  */
 public interface DataSearchService {
     void addVideoSummary(VideoSummary videoSummary);
+    void addVideoSummaryList(List<VideoSummary> list);
     void updateVideoSummary(VideoSummary videoSummary);
     void deleteVideoSummary(String videoId);
-    PageList<VideoSummary> searchVideo(String keyword, String nextIdStr, int pageNumber);
+    PageList<VideoSummary> searchVideo(String keyword, List<Integer> scopes, int pageNumber);
 }

search/search-service/src/main/java/cn/reghao/tnb/search/app/es/DocumentService.java

@@ -127,6 +127,22 @@ public class DocumentService {
         String jsonResult = indexResponse.result().jsonValue();
     }
 
+    public void batchAddText(String indexName, List<VideoText> videoTextList) {
+        videoTextList.forEach(videoText -> {
+            videoText.setId(""+idGenerator.nextId());
+        });
+
+        List<BulkOperation> bulkOperations = new ArrayList<>();
+        videoTextList.forEach(p -> bulkOperations.add(BulkOperation.of(b -> b.index(c -> c.id(p.getId()).document(p)))));
+        try {
+            BulkResponse bulkResponse = esClient.bulk(s -> s.index(indexName).operations(bulkOperations));
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        //bulkResponse.items().forEach(b -> log.info("bulk response result = {}", b.result()));
+        //log.error("bulk response.error() = {}", bulkResponse.errors());
+    }
+
     public void updateVideoText(String index, VideoText videoText) throws IOException {
         String id = videoText.getId();
         IndexResponse response = esClient.index(i -> i.index(index).document(videoText).id(id));

search/search-service/src/main/java/cn/reghao/tnb/search/app/es/QueryService.java

@@ -1,5 +1,6 @@
 package cn.reghao.tnb.search.app.es;
 
+import cn.reghao.tnb.content.api.constant.PostScope;
 import co.elastic.clients.elasticsearch.ElasticsearchClient;
 import co.elastic.clients.elasticsearch._types.FieldValue;
 import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
@@ -119,9 +120,20 @@ public class QueryService<T> {
         return Page.empty();
     }
 
-    public Page<T> queryWithHighlight(String index, Query query, int pn, int ps, Class<T> clazz) {
-        String highlightFieldName = "title";
+    public Page<T> queryByPage(String index, String queryString, int pn, int ps, Class<T> clazz) {
+        String highlightFieldName = "caseName";
+        List<String> otherFiledNames = List.of("caseName", "cause", "parties");
+
         // 1.构建查询的对象
+        QueryStringQuery stringQuery = new QueryStringQuery.Builder()
+                // 查询的字段
+                .fields(highlightFieldName, otherFiledNames.toArray(new String[0]))
+                .query(queryString)
+                .build();
+        Query query = new Query.Builder()
+                .queryString(stringQuery)
+                .build();
+
         // 2.高亮显示
         HighlightField highlightField = new HighlightField.Builder()
                 .matchedFields(highlightFieldName)
@@ -139,6 +151,95 @@ public class QueryService<T> {
                 .index(index)
                 .from(start)
                 .size(ps)
+                //.query(query)
+                //.highlight(highlight)
+                .build();
+        try {
+            SearchResponse<T> searchResponse = esClient.search(searchRequest, clazz);
+            HitsMetadata<T> hitsMetadata = searchResponse.hits();
+            long total = hitsMetadata.total().value();
+            List<T> list = hitsMetadata.hits().stream().map(mapper -> {
+                Map<String, List<String>> highlightMap = mapper.highlight();
+                String highlightStr = "";
+                if (!highlightMap.isEmpty()) {
+                    highlightStr = mapper.highlight()
+                            .get(highlightFieldName)
+                            .get(0);
+                }
+
+                T t = mapper.source();
+                try {
+                    Field field = clazz.getDeclaredField(highlightFieldName);
+                    field.setAccessible(true);
+
+                    if (t == null) {
+                        Object object = clazz.getDeclaredConstructors()[0].newInstance();
+                        field.set(object, highlightStr);
+                    } else {
+                        field.set(t, highlightStr);
+                    }
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+                return mapper.source();
+            }).collect(Collectors.toList());
+
+            //return list;
+            PageRequest pageRequest = PageRequest.of(pn-1, ps);
+            return new PageImpl<>(list, pageRequest, total);
+        } catch (IOException e) {
+            log.error("search By Query Highlight error", e);
+        }
+
+        return Page.empty();
+    }
+
+    public Page<T> queryWithHighlight(String indexName, String fieldName, String fieldValue, int pn, int ps, Class<T> clazz) {
+        List<Integer> scopes = List.of();
+        int scope1 = PostScope.PROTECT.getCode();
+        int scope2 = PostScope.PUBLIC.getCode();
+
+        QueryStringQuery stringQuery = new QueryStringQuery.Builder()
+                // 查询的字段
+                .fields(fieldName)
+                .query(fieldValue)
+                .build();
+
+        BoolQuery.Builder builder = new BoolQuery.Builder();
+        for (int scope : scopes) {
+            builder.should(m -> m.term(t -> t.field("scope").value(FieldValue.of(scope))));
+        }
+        builder.must(m -> m.queryString(stringQuery));
+        BoolQuery boolQuery = builder.build();
+        Query query = new Query.Builder()
+                .bool(boolQuery)
+                .build();
+
+        Query query1 = new Query.Builder()
+                .bool(BoolQuery.of(b -> b
+                        .must(m -> m.term(t -> t.field("scope").value(FieldValue.of(scope2))))
+                        .must(m -> m.queryString(stringQuery))))
+                .build();
+
+        String highlightFieldName = fieldName;
+        // 1.构建查询的对象
+        // 2.高亮显示
+        HighlightField highlightField = new HighlightField.Builder()
+                .matchedFields(highlightFieldName)
+                .preTags("<span style=\"color:red\">")
+                .postTags("</span>")
+                .build();
+        Highlight highlight = new Highlight.Builder()
+                .fields(highlightFieldName, highlightField)
+                .requireFieldMatch(false)
+                .build();
+
+        // 3.搜索请求
+        int start = (pn-1)*ps;
+        SearchRequest searchRequest = new SearchRequest.Builder()
+                .index(indexName)
+                .from(start)
+                .size(ps)
                 .query(query)
                 .highlight(highlight)
                 .build();

search/search-service/src/main/java/cn/reghao/tnb/search/app/es/VideoTextQuery.java

@@ -0,0 +1,117 @@
+package cn.reghao.tnb.search.app.es;
+
+import cn.reghao.tnb.search.app.model.po.VideoText;
+import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.elasticsearch._types.FieldValue;
+import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.Query;
+import co.elastic.clients.elasticsearch._types.query_dsl.QueryStringQuery;
+import co.elastic.clients.elasticsearch.core.SearchRequest;
+import co.elastic.clients.elasticsearch.core.SearchResponse;
+import co.elastic.clients.elasticsearch.core.search.Highlight;
+import co.elastic.clients.elasticsearch.core.search.HighlightField;
+import co.elastic.clients.elasticsearch.core.search.HitsMetadata;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageImpl;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * @author reghao
+ * @date 2025-03-12 11:16:48
+ */
+@Slf4j
+@Service
+public class VideoTextQuery {
+    private String indexName = "videotext";
+    private final ElasticsearchClient esClient;
+
+    public VideoTextQuery(ElasticService elasticService) {
+        this.esClient = elasticService.getElasticsearchClient();
+    }
+
+    public Page<VideoText> queryWithHighlight(String keyword, List<Integer> scopes, int pn, int ps) {
+        String fieldName = "title";
+        String fieldValue = keyword;
+        QueryStringQuery stringQuery = new QueryStringQuery.Builder()
+                // 查询的字段
+                .fields(fieldName)
+                .query(fieldValue)
+                .build();
+
+        BoolQuery.Builder builder = new BoolQuery.Builder();
+        for (int scope : scopes) {
+            builder.should(m -> m.term(t -> t.field("scope").value(FieldValue.of(scope))));
+        }
+        builder.must(m -> m.queryString(stringQuery));
+        BoolQuery boolQuery = builder.build();
+        Query query = new Query.Builder()
+                .bool(boolQuery)
+                .build();
+        Query query1 = new Query.Builder()
+                .bool(BoolQuery.of(b -> b
+                        .must(m -> m.term(t -> t.field("scope").value(FieldValue.of(1))))
+                        .must(m -> m.queryString(stringQuery))))
+                .build();
+
+        // 1.构建查询的对象
+        // 2.高亮显示
+        String highlightFieldName = fieldName;
+        HighlightField highlightField = new HighlightField.Builder()
+                .matchedFields(highlightFieldName)
+                .preTags("<span style=\"color:red\">")
+                .postTags("</span>")
+                .build();
+        Highlight highlight = new Highlight.Builder()
+                .fields(highlightFieldName, highlightField)
+                .requireFieldMatch(false)
+                .build();
+
+        // 3.搜索请求
+        int start = (pn-1)*ps;
+        SearchRequest searchRequest = new SearchRequest.Builder()
+                .index(indexName)
+                .from(start)
+                .size(ps)
+                .query(query)
+                .highlight(highlight)
+                .build();
+        try {
+            SearchResponse<VideoText> searchResponse = esClient.search(searchRequest, VideoText.class);
+            HitsMetadata<VideoText> hitsMetadata = searchResponse.hits();
+            long total = hitsMetadata.total().value();
+            List<VideoText> list = hitsMetadata.hits().stream().map(mapper -> {
+                Map<String, List<String>> highlightMap = mapper.highlight();
+                String highlightStr = "";
+                if (!highlightMap.isEmpty()) {
+                    highlightStr = mapper.highlight()
+                            .get(highlightFieldName)
+                            .get(0);
+                }
+
+                VideoText videoText = mapper.source();
+                try {
+                    Class<?> clazz = videoText.getClass();
+                    Field field = clazz.getDeclaredField(highlightFieldName);
+                    field.setAccessible(true);
+                    field.set(videoText, highlightStr);
+                } catch (Exception e) {
+                    log.error(e.getMessage());
+                }
+                return mapper.source();
+            }).collect(Collectors.toList());
+            return new PageImpl<>(list, PageRequest.of(pn-1, ps), total);
+        } catch (IOException e) {
+            log.error("search By Query Highlight error: {}", e.getMessage());
+        }
+
+        return Page.empty();
+    }
+}

search/search-service/src/main/java/cn/reghao/tnb/search/app/log/NginxLogDocumentService.java → search/search-service/src/main/java/cn/reghao/tnb/search/app/log/NginxLogDocument.java

@@ -22,11 +22,11 @@ import java.util.List;
  */
 @Slf4j
 @Service
-public class NginxLogDocumentService {
+public class NginxLogDocument {
     private final ElasticsearchClient esClient;
     private final SnowFlake idGenerator;
 
-    public NginxLogDocumentService(ElasticService elasticService) {
+    public NginxLogDocument(ElasticService elasticService) {
         this.esClient = elasticService.getElasticsearchClient();
         this.idGenerator = new SnowFlake(1, 1);
     }

search/search-service/src/main/java/cn/reghao/tnb/search/app/log/NginxLogSearchService.java → search/search-service/src/main/java/cn/reghao/tnb/search/app/log/NginxLogSearch.java

@@ -32,11 +32,11 @@ import java.util.stream.Collectors;
  */
 @Slf4j
 @Service
-public class NginxLogSearchService {
+public class NginxLogSearch {
     private final ElasticsearchClient esClient;
     private final String indexName = "NginxLog";
 
-    public NginxLogSearchService(ElasticService elasticService) throws Exception {
+    public NginxLogSearch(ElasticService elasticService) throws Exception {
         this.esClient = elasticService.getElasticsearchClient();
     }
 

search/search-service/src/main/java/cn/reghao/tnb/search/app/log/NginxLogService.java

@@ -182,8 +182,8 @@ public class NginxLogService {
         IndexService indexService = new IndexService(elasticService);
         MappingService mappingService = new MappingService();
 
-        NginxLogDocumentService nginxLogDocumentService = new NginxLogDocumentService(elasticService);
-        NginxLogSearchService nginxLogSearchService = new NginxLogSearchService(elasticService);
+        NginxLogDocument nginxLogDocument = new NginxLogDocument(elasticService);
+        NginxLogSearch nginxLogSearch = new NginxLogSearch(elasticService);
         QueryService<NginxLog> queryService = new QueryService<>(elasticService);
         String index = "nginx_log";
 
@@ -214,7 +214,7 @@ public class NginxLogService {
         //String queryString = "content";
         //List<NginxLog> list = queryService.queryWithHighlight(index, queryString, pn, ps, NginxLog.class);
         //searchService.searchAll(index);
-        nginxLogSearchService.aggregate();
+        nginxLogSearch.aggregate();
         //searchService.count(index);
     }
 }

search/search-service/src/main/java/cn/reghao/tnb/search/app/rpc/DataSearchServiceImpl.java

@@ -4,15 +4,11 @@ import cn.reghao.tnb.search.api.dto.VideoSummary;
 import cn.reghao.tnb.search.api.iface.DataSearchService;
 import cn.reghao.tnb.search.app.es.DocumentService;
 import cn.reghao.tnb.search.app.es.ElasticService;
-import cn.reghao.tnb.search.app.es.QueryService;
+import cn.reghao.tnb.search.app.es.VideoTextQuery;
 import cn.reghao.tnb.search.app.lucene.LuceneIndex;
 import cn.reghao.tnb.search.app.lucene.LuceneQuery;
 import cn.reghao.tnb.search.app.model.po.VideoText;
 import cn.reghao.jutil.jdk.db.PageList;
-import co.elastic.clients.elasticsearch._types.FieldValue;
-import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
-import co.elastic.clients.elasticsearch._types.query_dsl.Query;
-import co.elastic.clients.elasticsearch._types.query_dsl.QueryStringQuery;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.dubbo.config.annotation.DubboService;
 import org.springframework.data.domain.Page;
@@ -35,18 +31,17 @@ import java.util.stream.Collectors;
 @DubboService
 @Service
 public class DataSearchServiceImpl implements DataSearchService {
-    private final String index = VideoText.class.getSimpleName().toLowerCase(Locale.ROOT);
-    private int ps = 20;
+    private final String indexName = VideoText.class.getSimpleName().toLowerCase(Locale.ROOT);
+    private int ps = 12;
     private final DocumentService documentService;
     private final LuceneIndex luceneIndex;
-    private final QueryService<VideoText> esQuery;
-    private final LuceneQuery luceneQuery;
+    private final VideoTextQuery videoTextQuery;
 
-    public DataSearchServiceImpl(ElasticService elasticService, LuceneQuery luceneQuery, LuceneIndex luceneIndex) {
-        this.esQuery = new QueryService<>(elasticService);
+    public DataSearchServiceImpl(ElasticService elasticService, LuceneQuery luceneQuery,
+                                 LuceneIndex luceneIndex, VideoTextQuery videoTextQuery) {
         this.documentService = new DocumentService(elasticService);
-        this.luceneQuery = luceneQuery;
         this.luceneIndex = luceneIndex;
+        this.videoTextQuery = videoTextQuery;
     }
 
     public void addVideoSummary(VideoSummary videoSummary) {
@@ -54,7 +49,7 @@ public class DataSearchServiceImpl implements DataSearchService {
         try {
             VideoText videoText = new VideoText(videoSummary);
             if (type == 1) {
-                documentService.addVideoText(index, videoText);
+                documentService.addVideoText(indexName, videoText);
             } else{
                 luceneIndex.createVideoTextIndex(videoText);
             }
@@ -63,6 +58,13 @@ public class DataSearchServiceImpl implements DataSearchService {
         }
     }
 
+    public void addVideoSummaryList(List<VideoSummary> list) {
+        List<VideoText> videoTextList = list.stream()
+                .map(VideoText::new)
+                .collect(Collectors.toList());
+        documentService.batchAddText(indexName, videoTextList);
+    }
+
     @Override
     public void updateVideoSummary(VideoSummary videoSummary) {
     }
@@ -72,30 +74,8 @@ public class DataSearchServiceImpl implements DataSearchService {
     }
 
     @Override
-    public PageList<VideoSummary> searchVideo(String keyword, String nextIdStr, int pn) {
-        QueryStringQuery stringQuery = new QueryStringQuery.Builder()
-                // 查询的字段
-                .fields("title")
-                .query(keyword)
-                .build();
-
-        boolean vip1 = Boolean.parseBoolean(nextIdStr);
-        Query query;
-        if (!vip1) {
-            query = new Query.Builder()
-                    .queryString(stringQuery)
-                    .build();
-        } else {
-            query = new Query.Builder()
-                    .bool(BoolQuery.of(b -> b
-                            .must(m -> m.term(t -> t.field("vip").value(FieldValue.of(vip1))))
-                            .must(m -> m.queryString(stringQuery))))
-                    .build();
-        }
-
-        Page<VideoText> page = esQuery.queryWithHighlight(index, query, pn, ps, VideoText.class);
-        //Page<VideoText> page = esQuery.queryWithHighlight(index, keyword, pn, ps, VideoText.class);
-        //Page<VideoText> page1 = luceneQuery.queryWithHighlight(index, keyword, pn, ps);
+    public PageList<VideoSummary> searchVideo(String keyword, List<Integer> scopes, int pn) {
+        Page<VideoText> page = videoTextQuery.queryWithHighlight(keyword, scopes, pn, ps);
         List<VideoSummary> list = page.stream().map(videoText -> {
             String videoId = videoText.getVideoId();
             String title = videoText.getTitle();

search/search-service/src/main/resources/application-dev.yml

@@ -21,7 +21,7 @@ spring:
     username: dev
     password: Dev@123456
 es:
-  host: 127.0.0.1
+  host: 192.168.0.10
   port: 9200
   username: elastic
   password: VLTtN03SSJ4lsyyg56kf

search/search-service/src/main/resources/application-test.yml

@@ -21,7 +21,7 @@ spring:
     username: test
     password: Test@123456
 es:
-  host: 127.0.0.1
+  host: 192.168.0.10
   port: 9200
   username: elastic
   password: VLTtN03SSJ4lsyyg56kf

search/search-service/src/test/java/SearchTest.java

@@ -2,6 +2,7 @@ import ch.qos.logback.classic.Level;
 import ch.qos.logback.classic.Logger;
 import ch.qos.logback.classic.LoggerContext;
 import cn.reghao.jutil.tool.id.SnowFlake;
+import cn.reghao.tnb.content.api.constant.PostScope;
 import cn.reghao.tnb.search.api.dto.VideoSummary;
 import cn.reghao.tnb.search.app.SearchApplication;
 import cn.reghao.tnb.search.app.es.*;
@@ -12,7 +13,11 @@ import cn.reghao.tnb.search.app.model.po.VideoText;
 import cn.reghao.tnb.search.app.model.po.Wenshu;
 import cn.reghao.tnb.search.app.util.ClassUtil;
 import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.elasticsearch._types.FieldValue;
 import co.elastic.clients.elasticsearch._types.mapping.Property;
+import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.Query;
+import co.elastic.clients.elasticsearch._types.query_dsl.QueryStringQuery;
 import co.elastic.clients.elasticsearch.indices.AnalyzeRequest;
 import co.elastic.clients.elasticsearch.indices.AnalyzeResponse;
 import co.elastic.clients.elasticsearch.indices.analyze.AnalyzeToken;
@@ -179,6 +184,7 @@ public class SearchTest {
         return null;
     }
 
+    @Test
     public void queryTest() {
         setLogLevel();
 
@@ -190,9 +196,6 @@ public class SearchTest {
             pn++;
             page = queryService.queryWithHighlight(index, queryString, pn, ps, Wenshu.class);
         }*/
-
-        System.out.println();
-
         Page<Wenshu> page = queryService.queryWithHighlight(index, queryString, pn, ps, Wenshu.class);
         long total = page.getTotalElements();
         int totalPages = page.getTotalPages();
@@ -200,7 +203,9 @@ public class SearchTest {
             page = queryService.queryWithHighlight(index, queryString, pn, ps, Wenshu.class);
             List<Wenshu> list = page.getContent();
             Wenshu wenshu = list.get(0);
-            Wenshu wenshu1 = queryService.queryById(Wenshu.class, index, wenshu.getId());
+            String id = wenshu.getId();
+            String caseName = wenshu.getCaseName();
+            System.out.println(caseName);
             pn++;
         }
     }
@@ -212,8 +217,6 @@ public class SearchTest {
     @Autowired
     LuceneDocument luceneDocument;
     @Autowired
-    QueryService<VideoSummary> esQuery;
-    @Autowired
     SearchService searchService;
     @Test
     public void storeTest() throws Exception {
@@ -242,17 +245,33 @@ public class SearchTest {
         //Page<VideoSummary> page2 = esQuery.queryWithHighlight(index, kw, pn, ps, VideoSummary.class);
         System.out.println();
 
-        Map<String, Property> propertyMap = mappingService.getVideoTextPropertyMap();
+        //Map<String, Property> propertyMap = mappingService.getVideoTextPropertyMap();
 //        indexService.deleteIndex(index);
 //        indexService.createIndex(index, propertyMap);
         //indexService.getIndex(index);
-        indexService.deleteIndex(index);
+        //indexService.deleteIndex(index);
         //indexService.getIndex(index);
         //indexService.updateMapping(index);
         //searchService.searchAll(index);
         //searchService.count(index);
     }
 
+    @Autowired
+    VideoTextQuery videoTextQuery;
+    @Test
+    public void searchTest() {
+        int pn = 1;
+        int ps = 1000;
+        String keyword = "大奶";
+
+        List<Integer> scopes = List.of(PostScope.PUBLIC.getCode(), PostScope.PROTECT.getCode());
+        Page<VideoText> page = videoTextQuery.queryWithHighlight(keyword, scopes, pn, ps);
+        //Page<VideoText> page1 = esQuery.queryByPage(index, "query", pn, ps, VideoText.class);
+        long total = page.getTotalElements();
+        List<VideoText> list = page.getContent();
+        System.out.println();
+    }
+
     @Test
     public void analyzerTest() throws IOException {
         String text = "中华人民共和国国歌";
@@ -263,8 +282,11 @@ public class SearchTest {
                 .text(text)
                 .analyzer(analyzer)
                 .build();
-        ElasticsearchClient esClient = elasticService.getElasticsearchClient();
+        /*ElasticsearchClient esClient = elasticService.getElasticsearchClient();
         AnalyzeResponse analyzeResponse = esClient.indices().analyze(analyzeRequest);
-        List<AnalyzeToken> tokens = analyzeResponse.tokens();
+        List<AnalyzeToken> tokens = analyzeResponse.tokens();*/
+
+        String index = "videotext";
+        documentService.deleteAllDocument(index);
     }
 }