将 dagent 和 dmaster 的日志统一存储到 mongodb

dagent 通过 MQTT 将日志发送到 dmaster

common/src/main/java/cn/reghao/autodop/common/log/Appenders.java

@@ -0,0 +1,55 @@
+package cn.reghao.autodop.common.log;
+
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.encoder.PatternLayoutEncoder;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.Appender;
+import ch.qos.logback.core.ConsoleAppender;
+import ch.qos.logback.core.FileAppender;
+import cn.reghao.autodop.common.mqtt.MosquittoProperties;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author reghao
+ * @date 2021-06-11 13:31:20
+ */
+public class Appenders {
+    private LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
+
+    public Appender<ILoggingEvent> mqttAppender(MosquittoProperties properties, String appId) {
+        PatternLayoutEncoder layoutEncoder = new PatternLayoutEncoder();
+        layoutEncoder.setPattern("%date %level [%thread] %logger{10} [%file:%line] %msg%n");
+        layoutEncoder.setContext(loggerContext);
+        layoutEncoder.start();
+
+        MqttAppender mqttAppender = new MqttAppender(properties, appId);
+        mqttAppender.setContext(loggerContext);
+        mqttAppender.start();
+        return mqttAppender;
+    }
+
+    public Appender<ILoggingEvent> fileAppender(LoggerContext loggerContext) {
+        PatternLayoutEncoder layoutEncoder = new PatternLayoutEncoder();
+        layoutEncoder.setPattern("%date %level [%thread] %logger{10} [%file:%line] %msg%n");
+        layoutEncoder.setContext(loggerContext);
+        layoutEncoder.start();
+
+        FileAppender<ILoggingEvent> fileAppender = new FileAppender<ILoggingEvent>();
+        fileAppender.setFile("file");
+        fileAppender.setEncoder(layoutEncoder);
+        fileAppender.setContext(loggerContext);
+        fileAppender.start();
+        return fileAppender;
+    }
+
+    public Appender<ILoggingEvent> consoleAppender(LoggerContext loggerContext) {
+        PatternLayoutEncoder layoutEncoder = new PatternLayoutEncoder();
+        layoutEncoder.setPattern("%d{HH:mm:ss.SSS} [%thread] %-5level %c %M %L - %msg%n");
+        layoutEncoder.setContext(loggerContext);
+        layoutEncoder.start();
+
+        ConsoleAppender<ILoggingEvent> consoleAppender = new ConsoleAppender<>();
+        consoleAppender.start();
+        return consoleAppender;
+    }
+}

common/src/main/java/cn/reghao/autodop/common/log/InternalMqttClient.java

@@ -1,6 +1,7 @@
 package cn.reghao.autodop.common.log;
 
 import org.eclipse.paho.client.mqttv3.*;
+import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence;
 import org.eclipse.paho.client.mqttv3.persist.MqttDefaultFilePersistence;
 
 /**
@@ -11,7 +12,8 @@ public class InternalMqttClient extends MqttClient implements MqttCallback {
     private MqttClientPersistence clientPersistence;
 
     public InternalMqttClient(String serverURI, String clientId) throws MqttException {
-        this(serverURI,clientId, new MqttDefaultFilePersistence());
+        //this(serverURI, clientId, new MqttDefaultFilePersistence());
+        this(serverURI, clientId, new MemoryPersistence());
     }
 
     public InternalMqttClient(String serverURI, String clientId, MqttClientPersistence persistence) throws MqttException {

common/src/main/java/cn/reghao/autodop/common/log/LoggerConfig.java

@@ -3,11 +3,8 @@ package cn.reghao.autodop.common.log;
 import ch.qos.logback.classic.Level;
 import ch.qos.logback.classic.Logger;
 import ch.qos.logback.classic.LoggerContext;
-import ch.qos.logback.classic.encoder.PatternLayoutEncoder;
 import ch.qos.logback.classic.spi.ILoggingEvent;
 import ch.qos.logback.core.Appender;
-import ch.qos.logback.core.ConsoleAppender;
-import ch.qos.logback.core.FileAppender;
 import org.slf4j.LoggerFactory;
 
 /**
@@ -19,51 +16,10 @@ import org.slf4j.LoggerFactory;
 public class LoggerConfig {
     private LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
 
-    public void initialize() {
+    public void addAppender(Appender<ILoggingEvent> appender) {
         Logger rootLogger = loggerContext.getLogger("ROOT");
         rootLogger.setAdditive(false);
         rootLogger.setLevel(Level.INFO);
-        //rootLogger.addAppender(amqpAppender(rabbitProperties));
-    }
-
-    // TODO dagent 运行日志全部通过 MQTT 发送到 dmaster
-    private Appender<ILoggingEvent> mqttAppender() {
-        PatternLayoutEncoder layoutEncoder = new PatternLayoutEncoder();
-        layoutEncoder.setPattern("%date %level [%thread] %logger{10} [%file:%line] %msg%n");
-        layoutEncoder.setContext(loggerContext);
-        layoutEncoder.start();
-
-        MqttAppender mqttAppender = new MqttAppender();
-        mqttAppender.setContext(loggerContext);
-        mqttAppender.start();
-
-        return mqttAppender;
-    }
-
-    private Appender<ILoggingEvent> fileAppender() {
-        PatternLayoutEncoder layoutEncoder = new PatternLayoutEncoder();
-        layoutEncoder.setPattern("%date %level [%thread] %logger{10} [%file:%line] %msg%n");
-        layoutEncoder.setContext(loggerContext);
-        layoutEncoder.start();
-
-        FileAppender<ILoggingEvent> fileAppender = new FileAppender<ILoggingEvent>();
-        fileAppender.setFile("file");
-        fileAppender.setEncoder(layoutEncoder);
-        fileAppender.setContext(loggerContext);
-        fileAppender.start();
-
-        return fileAppender;
-    }
-
-    private Appender<ILoggingEvent> consoleAppender() {
-        PatternLayoutEncoder layoutEncoder = new PatternLayoutEncoder();
-        layoutEncoder.setPattern("%d{HH:mm:ss.SSS} [%thread] %-5level %c %M %L - %msg%n");
-        layoutEncoder.setContext(loggerContext);
-        layoutEncoder.start();
-
-        ConsoleAppender<ILoggingEvent> consoleAppender = new ConsoleAppender<>();
-        consoleAppender.start();
-
-        return consoleAppender;
+        rootLogger.addAppender(appender);
     }
 }

common/src/main/java/cn/reghao/autodop/common/log/MqttAppender.java

@@ -1,15 +1,18 @@
 package cn.reghao.autodop.common.log;
 
 import ch.qos.logback.classic.spi.ILoggingEvent;
-import ch.qos.logback.classic.spi.LoggingEvent;
 import ch.qos.logback.core.UnsynchronizedAppenderBase;
+import cn.reghao.autodop.common.message.AsyncMsg;
+import cn.reghao.autodop.common.message.MessageType;
+import cn.reghao.autodop.common.message.ops.DagentOps;
+import cn.reghao.autodop.common.mqtt.MosquittoProperties;
+import cn.reghao.autodop.common.utils.DateTimeConverter;
+import cn.reghao.autodop.common.utils.MachineId;
+import cn.reghao.autodop.common.utils.serializer.JsonConverter;
 import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
 import org.eclipse.paho.client.mqttv3.MqttException;
 import org.eclipse.paho.client.mqttv3.MqttMessage;
 
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.Date;
 import java.util.UUID;
 
 /**
@@ -17,25 +20,28 @@ import java.util.UUID;
  * @date 2021-06-08 19:37:21
  */
 public class MqttAppender extends UnsynchronizedAppenderBase<ILoggingEvent> {
-    private static final String MQTT_URL = "tcp://localhost:1883";
-    private static final String MQTT_CLIENT_ID = UUID.randomUUID().toString();
-    private static final String MQTT_TOPIC = "mqttlogger";
-    private static final String MQTT_USER = "dev";
-    private static final String MQTT_PASSWORD = "Dev@123456";
-
+    private MosquittoProperties properties;
+    private String clientId = UUID.randomUUID().toString();
     private InternalMqttClient client;
+    private String machineId;
+    private String machineIpv4;
+    private String appId;
 
-    public MqttAppender() {
+    public MqttAppender(MosquittoProperties properties, String appId) {
+        this.properties = properties;
+        this.machineId = MachineId.id();
+        this.machineIpv4 = MachineId.ipv4();
+        this.appId = appId;
     }
 
     @Override
     public void start() {
         super.start();
         try {
-            client = new InternalMqttClient(MQTT_URL, MQTT_CLIENT_ID);
+            client = new InternalMqttClient(properties.getBroker(), clientId);
             MqttConnectOptions options = new MqttConnectOptions();
-            options.setUserName(MQTT_USER);
-            options.setPassword(MQTT_PASSWORD.toCharArray());
+            options.setUserName(properties.getUsername());
+            options.setPassword(properties.getPassword().toCharArray());
             client.setCallback(client);
             client.connect(options);
         } catch (MqttException e) {
@@ -55,32 +61,30 @@ public class MqttAppender extends UnsynchronizedAppenderBase<ILoggingEvent> {
 
     @Override
     protected void append(ILoggingEvent event) {
-        String json = format(event);
+        String json = JsonConverter.objectToJson(runtimeLog(event));
+        AsyncMsg asyncMsg = AsyncMsg.asyncMsg(MessageType.dagentType.name(), DagentOps.dagentLog.name(), json);
+        String payload = JsonConverter.objectToJson(asyncMsg);
+
         MqttMessage message = new MqttMessage();
         message.setQos(0);
-        message.setPayload(json.getBytes());
+        message.setPayload(payload.getBytes());
         try {
-            client.publish(MQTT_TOPIC, message);
+            client.publish(properties.getTopic(), message);
         } catch (MqttException e) {
             e.printStackTrace();
         }
     }
 
-    private final int DEFAULT_BUFFER_SIZE = 512;
-    private DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SZ");
-
-    private String format(ILoggingEvent event) {
-        StringBuilder buf = new StringBuilder(DEFAULT_BUFFER_SIZE);
-
-        buf.append(event.getFormattedMessage());
-        buf.append("\t");
-        buf.append(df.format(new Date(event.getTimeStamp())));
-        buf.append("\t");
-        buf.append(event.getLoggerName());
-        buf.append("\t");
-        buf.append(event.getThreadName());
-        buf.append("\t");
-        buf.append(event.getLevel().toString());
-        return buf.toString();
+    private RunningLog runtimeLog(ILoggingEvent event) {
+        RunningLog runningLog = new RunningLog();
+        runningLog.setMachineId(machineId);
+        runningLog.setMachineIpv4(machineIpv4);
+        runningLog.setAppId(appId);
+        runningLog.setLogTime(DateTimeConverter.localDateTime(event.getTimeStamp()));
+        runningLog.setLevel(event.getLevel().toString());
+        runningLog.setThreadName(event.getThreadName());
+        runningLog.setLoggerName(event.getLoggerName());
+        runningLog.setMessage(event.getMessage());
+        return runningLog;
     }
 }

common/src/main/java/cn/reghao/autodop/common/log/RunningLog.java

@@ -0,0 +1,23 @@
+package cn.reghao.autodop.common.log;
+
+import lombok.Data;
+
+import java.time.LocalDateTime;
+
+/**
+ * 应用运行日志
+ *
+ * @author reghao
+ * @date 2021-06-11 09:48:14
+ */
+@Data
+public class RunningLog {
+    private String machineId;
+    private String machineIpv4;
+    private String appId;
+    private LocalDateTime logTime;
+    private String level;
+    private String threadName;
+    private String loggerName;
+    private String message;
+}

common/src/main/java/cn/reghao/autodop/common/mqtt/MosquittoProperties.java

@@ -19,4 +19,6 @@ public class MosquittoProperties {
     private String username;
     @Value("${mosquitto.password}")
     private String password;
+    @Value("${mosquitto.topic}")
+    private String topic;
 }

common/src/main/java/cn/reghao/autodop/common/mqtt/MqttPub.java

@@ -48,22 +48,4 @@ public class MqttPub implements AutoCloseable {
         deliveryToken.setActionCallback(new PubActionListener());
         client.disconnect();
     }
-
-    public static void main(String[] args) {
-        MosquittoProperties properties = new MosquittoProperties();
-        properties.setBroker("tcp://192.168.0.220:1883");
-        properties.setUsername("test");
-        properties.setPassword("Test@123456");
-
-        String topic = "mqtt/sub";
-        try (MqttPub mqttPub = new MqttPub(properties)) {
-            for (int i = 0; i < 60; i++) {
-                String payload = "mqtt test msg " + i;
-                mqttPub.pub(topic, payload);
-                Thread.sleep(1_000);
-            }
-        } catch (MqttException | InterruptedException e) {
-            e.printStackTrace();
-        }
-    }
 }

dagent/src/main/java/cn/reghao/autodop/dagent/app/DockerAppServiceImpl.java

@@ -125,6 +125,9 @@ public class DockerAppServiceImpl implements AppService {
     public AppStatus status(String appId) throws DockerException {
         try (Docker docker = new Docker()) {
             String containerId = docker.getIdByName(appId);
+            if (containerId == null) {
+                throw new DockerException("没有和 " + appId + " 关联的容器");
+            }
             ContainerInfo containerInfo = docker.inspectContainer(containerId);
             return getAppStatus(appId, containerInfo);
         }

dagent/src/main/java/cn/reghao/autodop/dagent/mqttsub/DmasterMsgDispatcher.java

@@ -68,7 +68,7 @@ public class DmasterMsgDispatcher implements MqttCallback {
                 default:
             }
         } catch (Exception e) {
-            log.error(e.getMessage());
+            log.error("MQTT 消息异常: {}", e.getMessage());
         }
     }
 

dagent/src/main/java/cn/reghao/autodop/dagent/mqttsub/processor/docker/DockerOpsProcessor.java

@@ -34,27 +34,32 @@ public class DockerOpsProcessor implements OpsProcessor {
         AsyncMsg asyncMsg;
         switch (AppOps.valueOf(ops)) {
             case appDeploy:
+                log.info("部署应用...");
                 asyncMsg = AsyncMsg.asyncMsg(MessageType.appType.name(), AppOps.appDeployResult.name(),
                         JsonConverter.objectToJson(app.deploy(payload)));
-                log.info("部署应用完成...");
                 break;
             case appStatus:
+                log.info("获取应用运行状态...");
                 asyncMsg = AsyncMsg.asyncMsg(MessageType.appType.name(), AppOps.appStatusResult.name(),
                         JsonConverter.objectToJson(app.status(payload)));
                 break;
             case appRestart:
+                log.info("重启应用...");
                 asyncMsg = AsyncMsg.asyncMsg(MessageType.appType.name(), AppOps.appRestartResult.name(),
                         JsonConverter.objectToJson(app.restart(payload)));
                 break;
             case appStop:
+                log.info("停止应用...");
                 asyncMsg = AsyncMsg.asyncMsg(MessageType.appType.name(), AppOps.appStopResult.name(),
                         JsonConverter.objectToJson(app.stop(payload)));
                 break;
             case appStart:
+                log.info("启动应用...");
                 asyncMsg = AsyncMsg.asyncMsg(MessageType.appType.name(), AppOps.appStartResult.name(),
                         JsonConverter.objectToJson(app.start(payload)));
                 break;
             case appLog:
+                log.info("获取应用日志...");
                 asyncMsg = AsyncMsg.asyncMsg(MessageType.appType.name(), AppOps.appLogResult.name(),
                         JsonConverter.objectToJson(app.log(payload)));
                 break;

dagent/src/main/java/cn/reghao/autodop/dagent/utils/DagentLifecycle.java

@@ -1,9 +1,11 @@
 package cn.reghao.autodop.dagent.utils;
 
+import cn.reghao.autodop.common.log.Appenders;
 import cn.reghao.autodop.common.message.AsyncMsg;
 import cn.reghao.autodop.common.message.MessageType;
 import cn.reghao.autodop.common.dagent.machine.Machine;
 import cn.reghao.autodop.common.message.ops.MachineOps;
+import cn.reghao.autodop.common.mqtt.MosquittoProperties;
 import cn.reghao.autodop.common.mqtt.MqttPub;
 import cn.reghao.autodop.common.mqtt.MqttSub;
 import cn.reghao.autodop.common.utils.serializer.JsonConverter;
@@ -28,6 +30,7 @@ import org.springframework.stereotype.Component;
 @Slf4j
 @Component
 public class DagentLifecycle implements ApplicationRunner, DisposableBean {
+    private MosquittoProperties properties;
     private DmasterMsgDispatcher dmasterMsgDispatcher;
     private MqttPub mqttPub;
     private MqttSub mqttSub;
@@ -35,11 +38,13 @@ public class DagentLifecycle implements ApplicationRunner, DisposableBean {
     private Machine machine;
     private String topic = "dagent@" + Machine.machineId();
 
-    public DagentLifecycle(DmasterMsgDispatcher dmasterMsgDispatcher,
+    public DagentLifecycle(MosquittoProperties properties,
+                           DmasterMsgDispatcher dmasterMsgDispatcher,
                            MqttPub mqttPub,
                            MqttSub mqttSub,
                            MachineScheduler machineScheduler,
                            Machine machine) {
+        this.properties = properties;
         this.dmasterMsgDispatcher = dmasterMsgDispatcher;
         this.mqttPub = mqttPub;
         this.mqttSub = mqttSub;
@@ -49,6 +54,7 @@ public class DagentLifecycle implements ApplicationRunner, DisposableBean {
 
     @Override
     public void run(ApplicationArguments args) throws Exception {
+        initLogger();
         log.info("Dagent 启动...");
         activate();
     }
@@ -59,13 +65,18 @@ public class DagentLifecycle implements ApplicationRunner, DisposableBean {
         log.info("Dagent 停止...");
     }
 
+    private void initLogger() {
+        Appenders appenders = new Appenders();
+        LoggerConfig loggerConfig = new LoggerConfig();
+        loggerConfig.addAppender(appenders.mqttAppender(properties, "dagent"));
+    }
+
     /**
      * @date 2019-09-26 下午5:23
      */
     private void activate() throws Exception {
         subAndPub();
         scheduledJobs();
-        initLogger();
         log.info("dagent 应用已启动...");
     }
 
@@ -83,9 +94,4 @@ public class DagentLifecycle implements ApplicationRunner, DisposableBean {
         machineScheduler.add(HeartbeatJob.class, "machine-heartbeat", "0/10 * * * * ?");
         machineScheduler.start();
     }
-
-    private void initLogger() {
-        LoggerConfig loggerConfig = new LoggerConfig();
-        loggerConfig.initialize();
-    }
 }

dagent/src/main/resources/application-dev.yml

@@ -1,4 +1,5 @@
 mosquitto:
   broker: tcp://localhost:1883
   username: dev
-  password: Dev@123456
+  password: Dev@123456
+  topic: dmaster

dagent/src/main/resources/application-prod.yml

@@ -1,4 +1,5 @@
 mosquitto:
   broker: tcp://localhost:1883
   username: dev
-  password: Dev@123456
+  password: Dev@123456
+  topic: dmaster

dagent/src/main/resources/application-test.yml

@@ -1,4 +1,5 @@
 mosquitto:
   broker: tcp://192.168.0.220:1883
   username: test
-  password: Test@123456
+  password: Test@123456
+  topic: dmaster

dmaster/src/main/java/cn/reghao/autodop/dmaster/DmasterApplication.java

@@ -14,7 +14,7 @@ import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
 //@EnableCaching
 @SpringBootApplication
 @EnableJpaRepositories
-@EntityScan({"cn.reghao.autodop.dmaster", "cn.reghao.autodop.common"})
+@EntityScan({"cn.reghao.autodop.dmaster"})
 @ComponentScan({"cn.reghao.autodop.dmaster", "cn.reghao.autodop.common"})
 public class DmasterApplication {
 	@Bean

dmaster/src/main/java/cn/reghao/autodop/dmaster/app/controller/AppStatusController.java

@@ -54,7 +54,7 @@ public class AppStatusController {
     }
 
     @ApiOperation(value = "刷新已部署应用状态")
-    @PostMapping("/refresh}")
+    @PostMapping(value = "/refresh", produces = MediaType.APPLICATION_JSON_VALUE)
     public ResponseEntity<String> refreshDeployedApps() {
         statusService.refreshAppStatus();
         return ResponseEntity.ok().body(WebBody.success());

dmaster/src/main/java/cn/reghao/autodop/dmaster/common/orm/BaseDocument.java

@@ -23,5 +23,7 @@ public class BaseDocument implements Serializable {
 
     public BaseDocument() {
         this.isDelete = false;
+        this.createTime = LocalDateTime.now();
+        this.updateTime = LocalDateTime.now();
     }
 }

dmaster/src/main/java/cn/reghao/autodop/dmaster/mqttsub/processor/app/AppOpsProcessor.java

@@ -84,7 +84,7 @@ public class AppOpsProcessor implements OpsProcessor {
         CallResult<AppStatus> callResult = JsonConverter.jsonToObject(payload, type);
         if (callResult.getCode() != 0) {
             // TODO 处理调用失败的情况
-            log.error("调用失败...");
+            log.error("调用失败, 原因: {}", callResult.getMsg());
             return;
         }
 

dmaster/src/main/java/cn/reghao/autodop/dmaster/mqttsub/processor/dagent/DagentOpsProcessor.java

@@ -1,13 +1,14 @@
 package cn.reghao.autodop.dmaster.mqttsub.processor.dagent;
 
+import cn.reghao.autodop.common.log.RunningLog;
 import cn.reghao.autodop.common.message.ops.DagentOps;
-import cn.reghao.autodop.dmaster.common.thread.ThreadPoolWrapper;
+import cn.reghao.autodop.common.utils.serializer.JsonConverter;
 import cn.reghao.autodop.common.message.ops.OpsProcessor;
+import cn.reghao.autodop.dmaster.sys.entity.AppRunningLog;
+import cn.reghao.autodop.dmaster.sys.repository.AppRunningLogRepository;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
 
-import java.util.concurrent.ExecutorService;
-
 /**
  * 分发处理 Dagent 相关的消息
  *
@@ -17,17 +18,18 @@ import java.util.concurrent.ExecutorService;
 @Slf4j
 @Component
 public class DagentOpsProcessor implements OpsProcessor {
-    private ExecutorService threadPool;
+    private AppRunningLogRepository runtimeLogRepository;
 
-    public DagentOpsProcessor() {
-        this.threadPool = ThreadPoolWrapper.threadPool("dagent-ops-processor");
+    public DagentOpsProcessor(AppRunningLogRepository runtimeLogRepository) {
+        this.runtimeLogRepository = runtimeLogRepository;
     }
 
     @Override
     public void process(String ops, String payload) {
         switch (DagentOps.valueOf(ops)) {
             case dagentLog:
-                log.error("处理 dagent 日志");
+                RunningLog runningLog = JsonConverter.jsonToObject(payload, RunningLog.class);
+                runtimeLogRepository.save(AppRunningLog.from(runningLog));
                 break;
             default:
                 log.error("DagentOps 中没有相应类型...");

dmaster/src/main/java/cn/reghao/autodop/dmaster/sys/controller/SysLogPageController.java

@@ -0,0 +1,53 @@
+package cn.reghao.autodop.dmaster.sys.controller;
+
+import cn.reghao.autodop.dmaster.sys.entity.AppRunningLog;
+import cn.reghao.autodop.dmaster.sys.repository.AppRunningLogRepository;
+import cn.reghao.autodop.dmaster.utils.db.PageList;
+import cn.reghao.autodop.dmaster.utils.db.PageSort;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.data.domain.Sort;
+import org.springframework.stereotype.Controller;
+import org.springframework.ui.Model;
+import org.springframework.web.bind.annotation.*;
+
+/**
+ * @author reghao
+ * @date 2021-04-04 21:24:18
+ */
+@Api(tags = "日志页面")
+@RequestMapping("/sys/log")
+@Controller
+public class SysLogPageController {
+    private AppRunningLogRepository runtimeLogRepository;
+
+    public SysLogPageController(AppRunningLogRepository runtimeLogRepository) {
+        this.runtimeLogRepository = runtimeLogRepository;
+    }
+
+    @ApiOperation(value = "运行日志页面")
+    @GetMapping("/running")
+    public String runningLogPage(Model model) {
+        PageRequest pageRequest = PageSort.pageRequest("createTime", Sort.Direction.DESC);
+        Page<AppRunningLog> page = runtimeLogRepository.findAll(pageRequest);
+        PageList<AppRunningLog> pageList = PageList.pageList(page);
+
+        model.addAttribute("page", page);
+        model.addAttribute("list", pageList.getList());
+        return "/sys/log/running";
+    }
+
+    @ApiOperation(value = "访问日志页面")
+    @GetMapping("/access")
+    public String accessLogPage(Model model) {
+        PageRequest pageRequest = PageSort.pageRequest("createTime", Sort.Direction.DESC);
+        Page<AppRunningLog> page = runtimeLogRepository.findAll(pageRequest);
+        PageList<AppRunningLog> pageList = PageList.pageList(page);
+
+        model.addAttribute("page", page);
+        model.addAttribute("list", pageList.getList());
+        return "/sys/log/running";
+    }
+}

dmaster/src/main/java/cn/reghao/autodop/dmaster/sys/entity/AppRunningLog.java

@@ -0,0 +1,42 @@
+package cn.reghao.autodop.dmaster.sys.entity;
+
+import cn.reghao.autodop.common.log.RunningLog;
+import cn.reghao.autodop.dmaster.common.orm.BaseDocument;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import org.springframework.data.mongodb.core.mapping.Document;
+
+import java.time.LocalDateTime;
+
+/**
+ * 应用运行时日志
+ *
+ * @author reghao
+ * @date 2020-03-10 11:22:05
+ */
+@Data
+@EqualsAndHashCode(callSuper = false)
+@Document("AppRuntimeLog")
+public class AppRunningLog extends BaseDocument {
+    private String machineId;
+    private String machineIpv4;
+    private String appId;
+    private LocalDateTime logTime;
+    private String level;
+    private String threadName;
+    private String loggerName;
+    private String message;
+
+    public static AppRunningLog from(RunningLog runningLog) {
+        AppRunningLog log = new AppRunningLog();
+        log.setMachineId(runningLog.getMachineId());
+        log.setMachineIpv4(runningLog.getMachineIpv4());
+        log.setAppId(runningLog.getAppId());
+        log.setLogTime(runningLog.getLogTime());
+        log.setLevel(runningLog.getLevel());
+        log.setThreadName(runningLog.getThreadName());
+        log.setLoggerName(runningLog.getLoggerName());
+        log.setMessage(runningLog.getMessage());
+        return log;
+    }
+}

dmaster/src/main/java/cn/reghao/autodop/dmaster/sys/repository/AppRunningLogRepository.java

@@ -0,0 +1,14 @@
+package cn.reghao.autodop.dmaster.sys.repository;
+
+import cn.reghao.autodop.dmaster.sys.entity.AppRunningLog;
+import org.springframework.data.mongodb.repository.MongoRepository;
+
+import java.util.List;
+
+/**
+ * @author reghao
+ * @date 2020-01-21 14:53:03
+ */
+public interface AppRunningLogRepository extends MongoRepository<AppRunningLog, String> {
+    List<AppRunningLog> findByMachineIdAndAppId(String machineId, String appId);
+}

dmaster/src/main/java/cn/reghao/autodop/dmaster/utils/db/PageSort.java

@@ -33,4 +33,13 @@ public class PageSort {
         Sort sort = Sort.by(sortDirection, order);
         return PageRequest.of(index-1, size, sort);
     }
+
+    public static PageRequest pageRequest(String orderBy, Sort.Direction sortDirection) {
+        int index = Integer.parseInt(HttpRequestUtil.getParameter("page", String.valueOf(1)));
+        int size = Integer.parseInt(HttpRequestUtil.getParameter("size", String.valueOf(DEFAULT_PAGE_SIZE)));
+        String order =  HttpRequestUtil.getParameter("orderBy", orderBy);
+        String direction = HttpRequestUtil.getParameter("direction", sortDirection.toString());
+        Sort sort = Sort.by(sortDirection, order);
+        return PageRequest.of(index-1, size, sort);
+    }
 }

dmaster/src/main/java/cn/reghao/autodop/dmaster/utils/lifecycle/AfterAppStart.java → dmaster/src/main/java/cn/reghao/autodop/dmaster/utils/lifecycle/DmasterLifecycle.java

@@ -2,6 +2,7 @@ package cn.reghao.autodop.dmaster.utils.lifecycle;
 
 import cn.reghao.autodop.common.dagent.machine.hardware.disk.Disk;
 import cn.reghao.autodop.common.dagent.machine.hardware.disk.DiskInfo;
+import cn.reghao.autodop.common.log.LoggerConfig;
 import cn.reghao.autodop.common.utils.ByteConverter;
 import cn.reghao.autodop.common.utils.ByteType;
 import cn.reghao.autodop.common.mqtt.MqttSub;
@@ -13,9 +14,12 @@ import cn.reghao.autodop.dmaster.common.config.LocalBuildDir;
 import cn.reghao.autodop.dmaster.monitor.service.HealthCheckJob;
 import cn.reghao.autodop.dmaster.monitor.service.MonitorScheduler;
 import cn.reghao.autodop.dmaster.mqttsub.DagentMsgDispatcher;
+import cn.reghao.autodop.dmaster.sys.repository.AppRunningLogRepository;
+import cn.reghao.autodop.dmaster.utils.log.Appenders;
 import lombok.extern.slf4j.Slf4j;
 import org.eclipse.paho.client.mqttv3.MqttException;
 import org.quartz.SchedulerException;
+import org.springframework.beans.factory.DisposableBean;
 import org.springframework.boot.ApplicationArguments;
 import org.springframework.boot.ApplicationRunner;
 import org.springframework.stereotype.Component;
@@ -32,7 +36,8 @@ import java.io.File;
  */
 @Slf4j
 @Component
-public class AfterAppStart implements ApplicationRunner {
+public class DmasterLifecycle implements ApplicationRunner, DisposableBean {
+    private AppRunningLogRepository logRepository;
     private DagentMsgDispatcher dagentMsgDispatcher;
     private MqttSub mqttSub;
     private MonitorScheduler monitorScheduler;
@@ -41,10 +46,12 @@ public class AfterAppStart implements ApplicationRunner {
     private String machineIpv4 = MachineId.ipv4();
     private ByteConverter convert = new ByteConverter();
 
-    public AfterAppStart(DagentMsgDispatcher dagentMsgDispatcher,
-                         MqttSub mqttSub,
-                         MonitorScheduler monitorScheduler,
-                         BuildDirRepository buildDirRepository) {
+    public DmasterLifecycle(AppRunningLogRepository logRepository,
+                            DagentMsgDispatcher dagentMsgDispatcher,
+                            MqttSub mqttSub,
+                            MonitorScheduler monitorScheduler,
+                            BuildDirRepository buildDirRepository) {
+        this.logRepository = logRepository;
         this.dagentMsgDispatcher = dagentMsgDispatcher;
         this.mqttSub = mqttSub;
         this.monitorScheduler = monitorScheduler;
@@ -53,11 +60,23 @@ public class AfterAppStart implements ApplicationRunner {
 
     @Override
     public void run(ApplicationArguments args) throws MqttException {
+        initLogger();
         initializeBuildDir();
         sub();
         log.info("autodop-master 初始化完成...");
     }
 
+    @Override
+    public void destroy() {
+        log.info("Dmaster 停止...");
+    }
+
+    private void initLogger() {
+        Appenders appenders = new Appenders();
+        LoggerConfig loggerConfig = new LoggerConfig();
+        loggerConfig.addAppender(appenders.mongoAppender("dmaster", logRepository));
+    }
+
     private void initializeBuildDir() {
         BuildDir buildDir = buildDirRepository.findByMachineId(machineId);
         if (buildDir == null) {

dmaster/src/main/java/cn/reghao/autodop/dmaster/utils/log/Appenders.java

@@ -0,0 +1,28 @@
+package cn.reghao.autodop.dmaster.utils.log;
+
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.encoder.PatternLayoutEncoder;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.Appender;
+import cn.reghao.autodop.dmaster.sys.repository.AppRunningLogRepository;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author reghao
+ * @date 2021-06-11 13:31:20
+ */
+public class Appenders {
+    private LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
+    
+    public Appender<ILoggingEvent> mongoAppender(String appId, AppRunningLogRepository logRepository) {
+        PatternLayoutEncoder layoutEncoder = new PatternLayoutEncoder();
+        layoutEncoder.setPattern("%date %level [%thread] %logger{10} [%file:%line] %msg%n");
+        layoutEncoder.setContext(loggerContext);
+        layoutEncoder.start();
+
+        MongoAppender mongoAppender = new MongoAppender(appId, logRepository);
+        mongoAppender.setContext(loggerContext);
+        mongoAppender.start();
+        return mongoAppender;
+    }
+}

dmaster/src/main/java/cn/reghao/autodop/dmaster/utils/log/MongoAppender.java

@@ -0,0 +1,55 @@
+package cn.reghao.autodop.dmaster.utils.log;
+
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.UnsynchronizedAppenderBase;
+import cn.reghao.autodop.common.utils.DateTimeConverter;
+import cn.reghao.autodop.common.utils.MachineId;
+import cn.reghao.autodop.dmaster.sys.entity.AppRunningLog;
+import cn.reghao.autodop.dmaster.sys.repository.AppRunningLogRepository;
+
+/**
+ * @author reghao
+ * @date 2021-06-08 19:37:21
+ */
+public class MongoAppender extends UnsynchronizedAppenderBase<ILoggingEvent> {
+    private String machineId;
+    private String machineIpv4;
+    private String appId;
+    private AppRunningLogRepository logRepository;
+
+    public MongoAppender(String appId, AppRunningLogRepository logRepository) {
+        this.machineId = MachineId.id();
+        this.machineIpv4 = MachineId.ipv4();
+        this.appId = appId;
+        this.logRepository = logRepository;
+    }
+
+    @Override
+    public void start() {
+        super.start();
+    }
+
+    @Override
+    public void stop() {
+        super.stop();
+    }
+
+    @Override
+    protected void append(ILoggingEvent event) {
+        AppRunningLog runningLog = runtimeLog(event);
+        logRepository.save(runningLog);
+    }
+
+    private AppRunningLog runtimeLog(ILoggingEvent event) {
+        AppRunningLog runningLog = new AppRunningLog();
+        runningLog.setMachineId(machineId);
+        runningLog.setMachineIpv4(machineIpv4);
+        runningLog.setAppId(appId);
+        runningLog.setLogTime(DateTimeConverter.localDateTime(event.getTimeStamp()));
+        runningLog.setLevel(event.getLevel().toString());
+        runningLog.setThreadName(event.getThreadName());
+        runningLog.setLoggerName(event.getLoggerName());
+        runningLog.setMessage(event.getMessage());
+        return runningLog;
+    }
+}

dmaster/src/main/resources/application-dev.yml

@@ -10,6 +10,7 @@ mosquitto:
   broker: tcp://localhost:1883
   username: dev
   password: Dev@123456
+  topic: dagent@
 #mosquitto:
 #  broker: tcp://192.168.0.220:1883
 #  username: test

dmaster/src/main/resources/application-prod.yml

@@ -10,6 +10,7 @@ mosquitto:
   broker: tcp://192.168.0.211:1883
   username: test
   password: Test@123456
+  topic: dagent@
 oss:
   host: http://static.reghao.icu
   key: minioadmin

dmaster/src/main/resources/application-test.yml

@@ -10,6 +10,7 @@ mosquitto:
   broker: tcp://192.168.0.220:1883
   username: test
   password: Test@123456
+  topic: dagent@
 oss:
   host: http://static.reghao.icu
   key: minioadmin

dmaster/src/main/resources/templates/app/status.html

@@ -53,7 +53,7 @@
                     <th class="sortable" data-field="isRunning">运行状态</th>
                     <th class="sortable" data-field="startTime">启动时间</th>
                     <th class="sortable" data-field="pid">PID</th>
-                    <th class="sortable" data-field="lastCheck">上次检查时间</th>
+                    <th class="sortable" data-field="lastCheck">上次健康检查</th>
                     <th>操作</th>
                 </tr>
                 </thead>
@@ -70,7 +70,7 @@
                     <td th:text="${item.isRunning}">运行状态</td>
                     <td th:text="${item.startTime}">启动时间</td>
                     <td th:text="${item.pid}">PID</td>
-                    <td th:text="${item.lastCheck}">上次检查时间</td>
+                    <td th:text="${item.lastCheck}">上次健康检查</td>
                     <td>
                         <a class="ajax-post" th:href="@{'/api/app/status/restart/'+${item.appId}+'/'+${item.machineId}}">重启</a>
                         <a class="ajax-post" th:href="@{'/api/app/status/stop/'+${item.appId}+'/'+${item.machineId}}">停止</a>

dmaster/src/main/resources/templates/machine/host.html

@@ -50,7 +50,7 @@
                     <th class="sortable" data-field="osVersion">内核版本</th>
                     <th class="sortable" data-field="cpus">CPU 数量</th>
                     <th class="sortable" data-field="totalMem">内存容量</th>
-                    <th class="sortable" data-field="lastCheckedTime">最近检查时间</th>
+                    <th class="sortable" data-field="lastCheckedTime">上次接收心跳</th>
                     <th>操作</th>
                 </tr>
                 </thead>
@@ -65,7 +65,7 @@
                     <td th:text="${item.osVersion}">内核版本</td>
                     <td th:text="${item.cpus}">CPU 数量</td>
                     <td th:text="${item.totalMem}">内存容量</td>
-                    <td th:text="${item.lastCheckedTime}">最近检查时间</td>
+                    <td th:text="${item.lastCheckedTime}">上次接收心跳</td>
                     <td>
                         <a class="open-popup" data-title="机器详细信息" th:attr="data-url=@{'/machine/host/detail/'+${item.machineId}}"
                            data-size="1200,600" href="#">详细</a>

dmaster/src/main/resources/templates/sys/log/dagent.html

@@ -0,0 +1,42 @@
+<!DOCTYPE html>
+<html xmlns:th="http://www.thymeleaf.org">
+<head th:replace="/common/template :: header(~{::title},~{::link},~{::style})"></head>
+
+<body class="timo-layout-page">
+<div class="layui-card">
+    <div class="layui-card-header timo-card-header">
+        <span><i class="fa fa-bars"></i> dagent 日志</span>
+        <i class="layui-icon layui-icon-refresh refresh-btn"></i>
+    </div>
+    <div class="layui-card-body">
+        <div class="timo-table-wrap">
+            <table class="layui-table timo-table">
+                <thead>
+                <tr>
+                    <th class="sortable" data-field="logTime">时间</th>
+                    <th class="sortable" data-field="level">级别</th>
+                    <th class="sortable" data-field="threadName">线程名</th>
+                    <th class="sortable" data-field="loggerName">logger 名</th>
+                    <th class="sortable" data-field="message">内容</th>
+                </tr>
+                </thead>
+                <tbody>
+                <tr th:each="item:${list}">
+                    <td th:text="${item.logTime}">时间</td>
+                    <td th:text="${item.level}">级别</td>
+                    <td th:text="${item.threadName}">线程名</td>
+                    <td th:text="${item.loggerName}">logger 名</td>
+                    <td th:text="${item.message}">内容</td>
+                </tr>
+                </tbody>
+            </table>
+        </div>
+        <div th:replace="/common/fragment :: page"></div>
+    </div>
+</div>
+
+<script th:replace="/common/template :: script"></script>
+<script type="text/javascript">
+</script>
+</body>
+</html>

dmaster/src/main/resources/templates/sys/log/running.html

@@ -0,0 +1,46 @@
+<!DOCTYPE html>
+<html xmlns:th="http://www.thymeleaf.org">
+<head th:replace="/common/template :: header(~{::title},~{::link},~{::style})"></head>
+
+<body class="timo-layout-page">
+<div class="layui-card">
+    <div class="layui-card-header timo-card-header">
+        <span><i class="fa fa-bars"></i> 运行日志</span>
+        <i class="layui-icon layui-icon-refresh refresh-btn"></i>
+    </div>
+    <div class="layui-card-body">
+        <div class="timo-table-wrap">
+            <table class="layui-table timo-table">
+                <thead>
+                <tr>
+                    <th class="sortable" data-field="machineIpv4">机器地址</th>
+                    <th class="sortable" data-field="appId">应用</th>
+                    <th class="sortable" data-field="logTime">时间</th>
+                    <th class="sortable" data-field="level">级别</th>
+                    <th class="sortable" data-field="threadName">线程名</th>
+                    <th class="sortable" data-field="loggerName">logger 名</th>
+                    <th class="sortable" data-field="message">内容</th>
+                </tr>
+                </thead>
+                <tbody>
+                <tr th:each="item:${list}">
+                    <td th:text="${item.machineIpv4}">机器地址</td>
+                    <td th:text="${item.appId}">应用</td>
+                    <td th:text="${item.logTime}">时间</td>
+                    <td th:text="${item.level}">级别</td>
+                    <td th:text="${item.threadName}">线程名</td>
+                    <td th:text="${item.loggerName}">logger 名</td>
+                    <td th:text="${item.message}">日志内容</td>
+                </tr>
+                </tbody>
+            </table>
+        </div>
+        <div th:replace="/common/fragment :: page"></div>
+    </div>
+</div>
+
+<script th:replace="/common/template :: script"></script>
+<script type="text/javascript">
+</script>
+</body>
+</html>