tnb/search/search-service/src/test/java/NginxLogTest.java

import cn.reghao.jutil.jdk.serializer.JsonConverter;
import cn.reghao.jutil.jdk.id.SnowFlake;
import cn.reghao.tnb.search.app.SearchApplication;
//import cn.reghao.tnb.search.app.es.DocumentService;
//import cn.reghao.tnb.search.app.es.QueryService;
//import cn.reghao.tnb.search.app.es.SearchService;
import cn.reghao.tnb.search.app.log.model.NginxLog;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ActiveProfiles;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.List;

/**
 * @author reghao
 * @date 2025-07-03 15:56:02
 */
@Slf4j
@ActiveProfiles("dev")
@SpringBootTest(classes = SearchApplication.class)
public class NginxLogTest {
    /*@Autowired
    QueryService<NginxLog> queryService;
    @Autowired
    SearchService searchService;
    @Autowired
    DocumentService documentService;

    static SnowFlake idGenerator = new SnowFlake(1, 1);
    static long total = 0L;
    public static void readFileFileChannel(File file, String index, DocumentService documentService) {
        List<String> lines = new ArrayList<>();
        List<NginxLog> nginxLogs = new ArrayList<>();
        try {
            FileInputStream fis = new FileInputStream(file);
            FileChannel fileChannel = fis.getChannel();

            // 10MB
            int capacity = 10*1024*1024;
            ByteBuffer byteBuffer = ByteBuffer.allocate(capacity);
            StringBuffer buffer = new StringBuffer();
            while(fileChannel.read(byteBuffer) != -1) {
                //读取后，将位置置为0，将limit置为容量, 以备下次读入到字节缓冲中，从0开始存储
                byteBuffer.clear();
                byte[] bytes = byteBuffer.array();

                String str = new String(bytes);
                buffer.append(str);
                String[] strArray = buffer.toString().split(System.lineSeparator());
                for (int i = 0; i < strArray.length-1; i++) {
                    try {
                        NginxLog nginxLog = JsonConverter.jsonToObject(strArray[i], NginxLog.class);
                        nginxLogs.add(nginxLog);
                    } catch (Exception e) {
                        lines.add(strArray[i]);
                        //e.printStackTrace();
                    }
                }

                String lastLine = strArray[strArray.length-1];
                if (!lastLine.endsWith("}")) {
                    buffer = new StringBuffer();
                    buffer.append(strArray[strArray.length-1]);
                }

                while (nginxLogs.size() > 10_000) {
                    //nginxLogs.forEach(nginxLog -> nginxLog.setId(idGenerator.nextId()+""));
                    //NginxLog nginxLog = nginxLogs.get(0);
                    //documentService.update(index, nginxLog);

                    //documentService.batchAddDocument(index, nginxLogs);
                    //log.info("save {} nginxLogs", nginxLogs.size());
                    total += nginxLogs.size();
                    nginxLogs.clear();
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            // TODO close 处理
        }
    }

    void addNginxLogs(String index) {
        String baseDir = "/home/reghao/work/azy/data/konglog";
        File dir = new File(baseDir);
        for (File file : dir.listFiles()) {
            readFileFileChannel(file, index, documentService);
            log.info("total -> {}", total);
        }
        log.info("total -> {}", total);
    }

    @Test
    public void addNginxLogTest() {
        String index = "nginx_log";
        addNginxLogs(index);
    }

    @Test
    public void nginxLogTest() throws Exception {
        String index = "nginx_log";
        int ps = 1000;
        int pn = 1;
        String queryString = "content";
        //Page<NginxLog> list = queryService.queryWithHighlight(index, queryString, pn, ps, NginxLog.class);
        //searchService.searchAll(index);
        searchService.aggregate(index);
        System.out.println();
    }*/
}