import cn.reghao.jutil.jdk.serializer.JsonConverter; import cn.reghao.jutil.jdk.id.SnowFlake; import cn.reghao.tnb.search.app.SearchApplication; //import cn.reghao.tnb.search.app.es.DocumentService; //import cn.reghao.tnb.search.app.es.QueryService; //import cn.reghao.tnb.search.app.es.SearchService; import cn.reghao.tnb.search.app.log.model.NginxLog; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.List; /** * @author reghao * @date 2025-07-03 15:56:02 */ @Slf4j @ActiveProfiles("dev") @SpringBootTest(classes = SearchApplication.class) public class NginxLogTest { /*@Autowired QueryService queryService; @Autowired SearchService searchService; @Autowired DocumentService documentService; static SnowFlake idGenerator = new SnowFlake(1, 1); static long total = 0L; public static void readFileFileChannel(File file, String index, DocumentService documentService) { List lines = new ArrayList<>(); List nginxLogs = new ArrayList<>(); try { FileInputStream fis = new FileInputStream(file); FileChannel fileChannel = fis.getChannel(); // 10MB int capacity = 10*1024*1024; ByteBuffer byteBuffer = ByteBuffer.allocate(capacity); StringBuffer buffer = new StringBuffer(); while(fileChannel.read(byteBuffer) != -1) { //读取后,将位置置为0,将limit置为容量, 以备下次读入到字节缓冲中,从0开始存储 byteBuffer.clear(); byte[] bytes = byteBuffer.array(); String str = new String(bytes); buffer.append(str); String[] strArray = buffer.toString().split(System.lineSeparator()); for (int i = 0; i < strArray.length-1; i++) { try { NginxLog nginxLog = JsonConverter.jsonToObject(strArray[i], NginxLog.class); nginxLogs.add(nginxLog); } catch (Exception e) { lines.add(strArray[i]); //e.printStackTrace(); } } String lastLine = strArray[strArray.length-1]; if (!lastLine.endsWith("}")) { buffer = new StringBuffer(); buffer.append(strArray[strArray.length-1]); } while (nginxLogs.size() > 10_000) { //nginxLogs.forEach(nginxLog -> nginxLog.setId(idGenerator.nextId()+"")); //NginxLog nginxLog = nginxLogs.get(0); //documentService.update(index, nginxLog); //documentService.batchAddDocument(index, nginxLogs); //log.info("save {} nginxLogs", nginxLogs.size()); total += nginxLogs.size(); nginxLogs.clear(); } } } catch (IOException e) { e.printStackTrace(); } finally { // TODO close 处理 } } void addNginxLogs(String index) { String baseDir = "/home/reghao/work/azy/data/konglog"; File dir = new File(baseDir); for (File file : dir.listFiles()) { readFileFileChannel(file, index, documentService); log.info("total -> {}", total); } log.info("total -> {}", total); } @Test public void addNginxLogTest() { String index = "nginx_log"; addNginxLogs(index); } @Test public void nginxLogTest() throws Exception { String index = "nginx_log"; int ps = 1000; int pn = 1; String queryString = "content"; //Page list = queryService.queryWithHighlight(index, queryString, pn, ps, NginxLog.class); //searchService.searchAll(index); searchService.aggregate(index); System.out.println(); }*/ }