添加一个 deployer 模块, 专门用来部署 manager 和 agent

deployer/bin/deployer.json

@@ -0,0 +1,5 @@
+{
+  "localDir": "/home/reghao/code/java/devops/manager/bin",
+  "remoteDir": "/opt/apps/devops-manager",
+  "serverFile": "/home/reghao/Downloads/servers1.csv"
+}

deployer/pom.xml

@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>devops</artifactId>
+        <groupId>cn.reghao.devops</groupId>
+        <version>1.0.0</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>deployer</artifactId>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+        <maven.compiler.source>11</maven.compiler.source>
+        <maven.compiler.target>11</maven.compiler.target>
+        <project.build.outputDir>${project.basedir}/bin</project.build.outputDir>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>cn.reghao.jutil</groupId>
+            <artifactId>jdk</artifactId>
+            <version>1.0.0-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>com.github.mwiede</groupId>
+            <artifactId>jsch</artifactId>
+            <version>0.2.16</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <finalName>devops-${project.artifactId}</finalName>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <filtering>true</filtering>
+                <includes>
+                    <include>logback.xml</include>
+                </includes>
+            </resource>
+        </resources>
+
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>3.2.0</version>
+                <configuration>
+                    <archive>
+                        <manifest>
+                            <mainClass>cn.reghao.devops.deployer.DeployApp</mainClass>
+                        </manifest>
+                    </archive>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                    <!-- 不设置此属性则生成的 jar 包名字会带有 jar-with-dependencies -->
+                    <appendAssemblyId>false</appendAssemblyId>
+                    <outputDirectory>${project.build.outputDir}</outputDirectory>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

deployer/src/main/java/cn/reghao/devops/deployer/DeployApp.java

@@ -0,0 +1,45 @@
+package cn.reghao.devops.deployer;
+
+import cn.reghao.devops.deployer.model.DeployConfig;
+import cn.reghao.devops.deployer.model.RemoteHost;
+import cn.reghao.devops.deployer.util.ConfigFile;
+import cn.reghao.devops.deployer.util.Sftp;
+import cn.reghao.jutil.jdk.serializer.JsonConverter;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.File;
+import java.util.List;
+
+/**
+ * @author reghao
+ * @date 2024-02-19 14:16:27
+ */
+@Slf4j
+public class DeployApp {
+    public static void main(String[] args) {
+        if (args.length != 1) {
+            log.error("必须指定配置文件...");
+            return;
+        }
+        String configFilePath = ConfigFile.configFilePath(args[0], DeployApp.class);
+        DeployConfig deployConfig = JsonConverter.jsonFileToObject(new File(configFilePath), DeployConfig.class);
+
+        String serverFile = deployConfig.getServerFile();
+        String local = deployConfig.getLocalDir();
+        String remoteDir = deployConfig.getRemoteDir();
+
+        log.info("start deploy devops apps");
+        Sftp sftp = new Sftp();
+        List<RemoteHost> remoteHosts = sftp.getRemoteHost(serverFile);
+        for (RemoteHost remoteHost : remoteHosts) {
+            try {
+                String host = remoteHost.getHost();
+                sftp.deploy(local, remoteDir, remoteHost);
+                log.info("deploy devops on {} done", host);
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+        }
+        log.info("deploy devops done");
+    }
+}

deployer/src/main/java/cn/reghao/devops/deployer/model/DeployConfig.java

@@ -0,0 +1,16 @@
+package cn.reghao.devops.deployer.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+/**
+ * @author reghao
+ * @date 2024-02-20 10:07:48
+ */
+@AllArgsConstructor
+@Getter
+public class DeployConfig {
+    private String localDir;
+    private String remoteDir;
+    private String serverFile;
+}

deployer/src/main/java/cn/reghao/devops/deployer/model/RemoteHost.java

@@ -0,0 +1,18 @@
+package cn.reghao.devops.deployer.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+/**
+ * @author reghao
+ * @date 2024-02-20 09:12:10
+ */
+@AllArgsConstructor
+@Getter
+public class RemoteHost {
+    private String host;
+    private int port;
+    private String username;
+    private String password;
+    private String prikeyPath;
+}

deployer/src/main/java/cn/reghao/devops/deployer/model/UserInfoImpl.java

@@ -0,0 +1,53 @@
+package cn.reghao.devops.deployer.model;
+
+import com.jcraft.jsch.UserInfo;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * @author reghao
+ * @date 2024-02-19 14:44:45
+ */
+/**
+ * ssh private key passphrase info
+ */
+@Slf4j
+public class UserInfoImpl implements UserInfo {
+    /**
+     * ssh private key passphrase
+     */
+    private String passphrase;
+
+    public UserInfoImpl (String passphrase) {
+        this.passphrase = passphrase;
+    }
+
+    @Override
+    public String getPassphrase() {
+        return passphrase;
+    }
+
+    @Override
+    public String getPassword() {
+        return null;
+    }
+
+    @Override
+    public boolean promptPassphrase(String s) {
+        return true;
+    }
+
+    @Override
+    public boolean promptPassword(String s) {
+        return false;
+    }
+
+    @Override
+    public boolean promptYesNo(String s) {
+        return true;
+    }
+
+    @Override
+    public void showMessage(String message) {
+        log.info ("SSH Message:{}", message);
+    }
+}

deployer/src/main/java/cn/reghao/devops/deployer/util/ConfigFile.java

@@ -0,0 +1,53 @@
+package cn.reghao.devops.deployer.util;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.File;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * @author reghao
+ * @date 2021-03-03 18:47:01
+ */
+@Slf4j
+public class ConfigFile {
+    public static String configFilePath(String arg, Class<?> clazz) {
+        String configFilePath = null;
+        File configFile = new File(arg);
+        if (!configFile.exists()) {
+            if (arg.startsWith("./")) {
+                String filename = arg.replace(".", "")
+                        .replace("/", "");
+                configFilePath = runningHome(clazz) + "/" + filename;
+            } else if (!arg.contains("/")) {
+                configFilePath = runningHome(clazz) + "/" + arg;
+            } else {
+                log.error("相对路径的配置文件必须以 ./configFile 或 configFile 形式指定...");
+            }
+        } else {
+            // 绝对路径
+            configFilePath = arg;
+        }
+        return configFilePath;
+    }
+    
+    /**
+     * jar 文件运行目录
+     *
+     * @param
+     * @return
+     * @date 2021-03-03 下午6:33
+     */
+    public static String runningHome(Class<?> clazz) {
+        URL url = clazz.getProtectionDomain().getCodeSource().getLocation();
+        String jarFilePath = URLDecoder.decode(url.getPath(), StandardCharsets.UTF_8);
+        if (jarFilePath.endsWith(".jar")) {
+            jarFilePath = jarFilePath.substring(0, jarFilePath.lastIndexOf("/") + 1);
+        }
+
+        File file = new File(jarFilePath);
+        return file.getAbsolutePath();
+    }
+}

deployer/src/main/java/cn/reghao/devops/deployer/util/Sftp.java

@@ -0,0 +1,233 @@
+package cn.reghao.devops.deployer.util;
+
+import cn.reghao.devops.deployer.model.RemoteHost;
+import cn.reghao.devops.deployer.model.UserInfoImpl;
+import cn.reghao.jutil.jdk.shell.ShellResult;
+import cn.reghao.jutil.jdk.text.TextFile;
+import com.jcraft.jsch.*;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+/**
+ * @author reghao
+ * @date 2024-02-19 11:38:54
+ */
+@Slf4j
+public class Sftp {
+    private final TextFile textFile = new TextFile();
+
+    public Session getSession(RemoteHost remoteHost) throws Exception {
+        String host = remoteHost.getHost();
+        int port = remoteHost.getPort();
+        String username = remoteHost.getUsername();
+        String password = remoteHost.getPassword();
+        String prikeyPath = remoteHost.getPrikeyPath();
+
+        JSch jsch = new JSch();
+        Session session;
+        if (port <= 0) {
+            //连接服务器，采用默认端口
+            session = jsch.getSession(username, host);
+        } else {
+            //采用指定的端口连接服务器
+            session = jsch.getSession(username, host, port);
+        }
+
+        //如果服务器连接不上，则抛出异常
+        if (session == null) {
+            throw new Exception("session is null");
+        }
+
+        if (password != null) {
+            //设置登陆主机的密码
+            session.setPassword(password);
+            //设置第一次登陆的时候提示，可选值：(ask | yes | no)
+            session.setConfig("StrictHostKeyChecking", "no");
+        } else if (prikeyPath != null) {
+            session.setConfig("PreferredAuthentications", "publickey");
+            session.setConfig("userauth.gssapi-with-mic", "no");
+            session.setConfig("StrictHostKeyChecking", "ask");
+            session.setUserInfo(new UserInfoImpl(""));
+            jsch.addIdentity(prikeyPath);
+
+            session.setConfig("UseDNS", "no");
+            session.setConfig("kex", "diffie-hellman-group1-sha1,"
+                    + "diffie-hellman-group-exchange-sha1,"
+                    + "diffie-hellman-group-exchange-sha256");
+        } else {
+            throw new Exception("password and private key not exist either");
+        }
+
+        session.connect(30_000);
+        return session;
+    }
+
+    public void upload(Session session, String local, String destDir) throws JSchException, SftpException, IOException {
+        // 创建 sftp 通信通道
+        Channel channel = session.openChannel("sftp");
+        channel.connect(5_000);
+        ChannelSftp sftp = (ChannelSftp) channel;
+        sftp.cd(destDir);
+        File localFile = new File(local);
+        if (localFile.isFile()) {
+            putFile(localFile, destDir, sftp);
+        } else {
+            for (File file : Objects.requireNonNull(localFile.listFiles())) {
+                if (file.isFile()) {
+                    putFile(file, destDir, sftp);
+                }
+            }
+        }
+
+        if (channel.isConnected()) {
+            channel.disconnect();
+        }
+    }
+
+    private void putFile(File localFile, String destDir, ChannelSftp sftp) throws IOException, SftpException {
+        String filename = localFile.getName();
+        String remoteFilePath = String.format("%s/%s", destDir, filename);
+        OutputStream outstream = sftp.put(remoteFilePath);
+
+        InputStream instream = new FileInputStream(localFile);
+        byte[] bytes = new byte[1024];
+        int n;
+        while ((n = instream.read(bytes)) != -1) {
+            outstream.write(bytes, 0, n);
+        }
+
+        outstream.flush();
+        outstream.close();
+        instream.close();
+    }
+
+    /**
+     * 在远程机器上创建目录(等价于 mkdir -p)
+     *
+     * @param
+     * @return
+     * @date 2024-02-20 09:20:11
+     */
+    public boolean mkdir(Session session, String remotePath) throws SftpException, JSchException {
+        boolean exist = true;
+        // 创建 sftp 通信通道
+        Channel channel = session.openChannel("sftp");
+        channel.connect(5_000);
+        ChannelSftp sftp = (ChannelSftp) channel;
+
+        List<String> list = new ArrayList<>();
+        String[] arr = remotePath.split("/");
+        for (int i = 0; i < arr.length; i++) {
+            String path = "/" + arr[i];
+            if (i-1 > 0) {
+                list.add(list.get(i-1) + path);
+            } else {
+                list.add(path);
+            }
+        }
+
+        for (String path : list) {
+            try {
+                sftp.stat(path);
+            } catch (SftpException e) {
+                exist = false;
+                sftp.mkdir(path);
+            }
+        }
+
+        return exist;
+    }
+
+    public ShellResult exec(Session session, String command) throws JSchException, IOException {
+        StringBuilder sb = new StringBuilder();
+        // 创建 exec 通信通道
+        ChannelExec channel = (ChannelExec) session.openChannel("exec");
+        channel.setCommand(command);
+        channel.setInputStream(null);
+        channel.setErrStream(System.err);
+        channel.connect(10_000);
+        InputStream input = channel.getInputStream();
+        try {
+            BufferedReader inputReader = new BufferedReader(new InputStreamReader(input));
+            String line;
+            while((line = inputReader.readLine()) != null) {
+                sb.append(line).append(System.lineSeparator());
+            }
+        } finally {
+            if (input != null) {
+                try {
+                    input.close();
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+
+        int statusCode = channel.getExitStatus();
+        ShellResult shellResult = new ShellResult(statusCode);
+        shellResult.setResult(sb.toString());
+
+        if (channel.isConnected()) {
+            channel.disconnect();
+        }
+
+        return shellResult;
+    }
+
+    public List<RemoteHost> getRemoteHost(String filePath) {
+        return textFile.read(filePath).stream().map(line -> {
+            String[] arr = line.split(",");
+            if (arr.length == 4) {
+                String host = arr[0];
+                int port = Integer.parseInt(arr[1]);
+                String username = arr[2];
+                String password = arr[3];
+                return new RemoteHost(host, port, username, password, null);
+            } else if (arr.length == 5) {
+                String host = arr[0];
+                int port = Integer.parseInt(arr[1]);
+                String username = arr[2];
+                String prikeyPath = arr[4];
+                return new RemoteHost(host, port, username, null, prikeyPath);
+            }
+
+            return null;
+        }).filter(Objects::nonNull).collect(Collectors.toList());
+    }
+
+    public void deploy(String localDir, String remoteDir, RemoteHost remoteHost) throws Exception {
+        Session session = getSession(remoteHost);
+        boolean exist = mkdir(session, remoteDir);
+        if (exist) {
+            String command = String.format("cd %s && sh shutdown.sh", remoteDir);
+            ShellResult shellResult = exec(session, command);
+            if (!shellResult.isSuccess()) {
+                log.info("shutdown application failed: {} - {}", shellResult.getExitCode(), shellResult.getResult());
+            } else {
+                log.info("shutdown application successfully");
+            }
+        } else {
+            log.info("remote dir {} created", remoteDir);
+        }
+
+        upload(session, localDir, remoteDir);
+        log.info("files uploaded");
+
+        String command = String.format("cd %s && sh start.sh", remoteDir);
+        ShellResult shellResult = exec(session, command);
+        if (!shellResult.isSuccess()) {
+            log.info("start application failed: {} - {}", shellResult.getExitCode(), shellResult.getResult());
+        } else {
+            log.info("start application successfully");
+        }
+
+        if (session.isConnected()) {
+            session.disconnect();
+        }
+    }
+}

pom.xml

@@ -12,6 +12,7 @@
         <module>manager</module>
         <module>agent</module>
         <module>logstash</module>
+        <module>deployer</module>
     </modules>
     <packaging>pom</packaging>