优化构建部署流程

dmaster/src/main/java/cn/reghao/autodop/dmaster/app/cache/OrchestrationCache.java

@@ -23,4 +23,9 @@ public class OrchestrationCache {
         AppOrchestration app = appRepository.findByAppId(appId);
         return app;
     }
+
+    public AppOrchestration findByRepoAndBranch(String repo, String branch) {
+        AppOrchestration app = appRepository.findByAppRepoAndBranch(repo, branch);
+        return app;
+    }
 }

dmaster/src/main/java/cn/reghao/autodop/dmaster/app/controller/WebhookController.java

@@ -1,32 +1,13 @@
 package cn.reghao.autodop.dmaster.app.controller;
 
 import cn.reghao.autodop.common.result.WebResult;
-import cn.reghao.autodop.dmaster.app.constant.AppEnv;
-import cn.reghao.autodop.dmaster.app.service.BuildService;
-import cn.reghao.autodop.dmaster.app.service.OssService;
+import cn.reghao.autodop.common.utils.JsonUtil;
 import cn.reghao.autodop.dmaster.app.service.build.BuildDispatcher;
-import cn.reghao.autodop.dmaster.app.vo.AppToBuild;
-import cn.reghao.autodop.dmaster.app.vo.PageList;
 import cn.reghao.autodop.dmaster.app.vo.log.BuildDeployLog;
-import cn.reghao.autodop.dmaster.app.vo.log.BuildLogVO;
-import cn.reghao.autodop.dmaster.app.vo.log.CommitLogVO;
-import cn.reghao.autodop.dmaster.app.vo.log.DeployLogVO;
+import com.google.gson.JsonObject;
 import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiImplicitParam;
-import io.swagger.annotations.ApiImplicitParams;
-import io.swagger.annotations.ApiOperation;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.data.domain.PageRequest;
-import org.springframework.data.domain.Sort;
-import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
-import org.tmatesoft.svn.core.SVNException;
-
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
 
 /**
  * @author reghao
@@ -43,41 +24,13 @@ public class WebhookController {
         this.buildDispatcher = buildDispatcher;
     }
 
-    @PostMapping("/pipeline/{appId}")
-    public String build1(@PathVariable("appId") String appId,
-                         @RequestHeader("X-Codeup-Event") String event,
-                         @RequestHeader("x-codeup-delivery") String deliveryId,
-                         @RequestBody String body) throws Exception {
-
-        if (event == null || deliveryId == null) {
-            return WebResult.fail("非 webhook 调用");
-        }
-
-        Set<String> apps = checkArgs(appId);
-        List<BuildDeployLog> buildDeployLogs;
-        if (apps.size() == 0) {
-            return WebResult.success("ok");
-        } else {
-            buildDeployLogs = buildDispatcher.dispatch(apps, true);
-            List<DeployLogVO> deployResultVOS = buildDeployLogs.stream()
-                    .filter(buildDeployLog -> buildDeployLog.getDeployLog() != null)
-                    .map(BuildDeployLog::deployLog)
-                    .collect(Collectors.toList());
-
-            return WebResult.success(deployResultVOS);
-        }
-    }
-
-    /**
-     * 检查参数
-     *
-     * @param
-     * @return
-     * @date 2019-12-02 下午2:36
-     */
-    private Set<String> checkArgs(String appId) {
-        String whiteSpace = "\\s+";
-        String[] array = appId.replaceAll(whiteSpace, "").split(",");
-        return new HashSet<>(Arrays.asList(array));
+    @PostMapping("/pipeline")
+    public String build(@RequestBody String body) throws Exception {
+        JsonObject jsonObject = JsonUtil.jsonObject(body);
+        String repo = jsonObject.get("repository").getAsJsonObject().get("url").getAsString();
+        String ref = jsonObject.get("ref").getAsString();
+        String branch = ref.substring(ref.lastIndexOf("/")+1);
+        BuildDeployLog buildDeployLog = buildDispatcher.dispatch(repo, branch, true);
+        return WebResult.success(buildDeployLog);
     }
 }

dmaster/src/main/java/cn/reghao/autodop/dmaster/app/repository/orchestration/AppOrchestrationRepository.java

@@ -17,6 +17,7 @@ public interface AppOrchestrationRepository extends JpaRepository<AppOrchestrati
     Page<AppOrchestration> findByEnv(String env, Pageable pageable);
     Page<AppOrchestration> findAllByEnableIsTrueAndIsDeleteIsFalseAndEnv(String env, Pageable pageable);
     AppOrchestration findByAppId(String appId);
+    AppOrchestration findByAppRepoAndBranch(String appRepo, String branch);
 
     @Modifying
     @Transactional

dmaster/src/main/java/cn/reghao/autodop/dmaster/app/service/build/AppBuildPipeline.java

@@ -36,24 +36,32 @@ public class AppBuildPipeline implements Callable<BuildDeployLog> {
     @Override
     public BuildDeployLog call() {
         String appId = appIntegrate.getApp().getAppId();
+        long start = System.currentTimeMillis();
         if (!appIntegrate.update()) {
             return appIntegrate.log();
         }
+        long updateTime = System.currentTimeMillis() - start;
         log.info("{} 更新代码完成...", appId);
 
+        start = System.currentTimeMillis();
         if (!appIntegrate.compile()) {
             return appIntegrate.log();
         }
+        long compileTime = System.currentTimeMillis() - start;
         log.info("{} 编译代码完成...", appId);
 
+        start = System.currentTimeMillis();
         if (!appIntegrate.pack()) {
             return appIntegrate.log();
         }
+        long packTime = System.currentTimeMillis() - start;
         log.info("{} 打包代码完成...", appId);
         buildSuccessfully();
 
         if (isDeploy) {
+            start = System.currentTimeMillis();
             appIntegrate.deploy();
+            long deployTime = System.currentTimeMillis() - start;
             log.info("{} 部署代码完成...", appId);
         }
 

dmaster/src/main/java/cn/reghao/autodop/dmaster/app/service/build/AppIntegrate.java

@@ -25,6 +25,8 @@ import cn.reghao.autodop.dmaster.app.service.tools.updater.GitImpl;
 import cn.reghao.autodop.dmaster.app.service.tools.updater.SvnImpl;
 import cn.reghao.autodop.dmaster.common.config.SysConfig;
 import cn.reghao.autodop.dmaster.common.exception.ExceptionUtil;
+import lombok.AllArgsConstructor;
+import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 
 import java.io.File;
@@ -180,7 +182,7 @@ public class AppIntegrate {
         try {
             String remote;
             String branch;
-            String local = SysConfig.localRepo + "/" + app.getAppId();;
+            String local = SysConfig.localRepo + "/" + app.getAppId();
             ProjOrchestration proj = app.getProj();
             if (proj != null) {
                 remote = proj.getProjRepo();
@@ -221,12 +223,12 @@ public class AppIntegrate {
 
     private AppUpdateStatus update0(String remote, String branch, String local) throws Exception {
         CommitLog current = codeUpdater.update(remote, branch, local);
-        if (lastCommitLog == null ||
-                current.getCommitTime() > lastCommitLog.getCommitTime() ||
-                app.isAlwaysBuild()) {
+        if (lastCommitLog == null
+                || current.getCommitTime() > lastCommitLog.getCommitTime()
+                || app.isAlwaysBuild()) {
 
             if (app.getProj() == null) {
-                // TODO 拉取应用依赖的其他仓库
+                // TODO 拉取应用依赖的其他仓库(若存在)
                 Set<String> deps = app.getDependencyRepos();
                 if (deps != null && !deps.isEmpty()) {
                     for (String repo : deps) {
@@ -292,4 +294,12 @@ public class AppIntegrate {
     public BuildDeployLog log() {
         return new BuildDeployLog(buildLog, deployLog);
     }
+
+    @Data
+    @AllArgsConstructor
+    static class AppUpdateStatus {
+        private AppOrchestration app;
+        private CommitLog lastCommitLog;
+        private boolean isUpdated;
+    }
 }

dmaster/src/main/java/cn/reghao/autodop/dmaster/app/service/build/AppUpdateStatus.java

@@ -1,18 +0,0 @@
-package cn.reghao.autodop.dmaster.app.service.build;
-
-import cn.reghao.autodop.dmaster.app.entity.orchestration.AppOrchestration;
-import cn.reghao.autodop.dmaster.app.service.tools.updater.CommitLog;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-
-/**
- * @author reghao
- * @date 2020-05-17 21:18:17
- */
-@Data
-@AllArgsConstructor
-public class AppUpdateStatus {
-    private AppOrchestration app;
-    private CommitLog lastCommitLog;
-    private boolean isUpdated;
-}

dmaster/src/main/java/cn/reghao/autodop/dmaster/app/service/build/BuildDispatcher.java

@@ -39,6 +39,26 @@ public class BuildDispatcher {
         threadPool.submit(logConsumer);
     }
 
+    public BuildDeployLog dispatch(String repo, String branch, boolean isDeploy) throws Exception {
+        AppOrchestration app = caching.findByRepoAndBranch(repo, branch);
+        String appId = app.getAppId();
+        if (!onBuilding.contains(appId)) {
+            onBuilding.add(appId);
+        } else {
+            log.info("{} 正在构建中...", appId);
+        }
+
+        // TODO AppIntegrate 可缓存，不需要每次请求都创建，但要注意缓存失效时机
+        AppIntegrate appIntegrate = new AppIntegrate(app, appDeployer);
+        Future<BuildDeployLog> future = threadPool.submit(new AppBuildPipeline(appIntegrate, isDeploy));
+        while (!future.isDone() && !future.isCancelled()) {
+            Thread.sleep(1_000);
+        }
+
+        BuildDeployLog buildDeployLog = future.get();
+        return buildDeployLog;
+    }
+
     /**
      * 根据项目分发应用
      *

dmaster/src/main/java/cn/reghao/autodop/dmaster/app/service/tools/updater/CodeUpdater.java

@@ -18,7 +18,13 @@ public interface CodeUpdater {
      */
     CommitLog update(String remote, String branch, String local) throws Exception;
 
-    // TODO 拉取指定版本的代码
+    /**
+     * // TODO 拉取指定版本的代码
+     *
+     * @param commitId 指定的版本
+     * @return
+     * @date 2020-09-08 下午6:18
+     */
     CommitLog update(String remote, String branch, String local, String commitId) throws Exception;
 
     /**

dmaster/src/main/java/cn/reghao/autodop/dmaster/app/service/tools/updater/GitImpl.java

@@ -26,7 +26,7 @@ import java.util.List;
 import java.util.Properties;
 
 /**
- * TODO git 实现
+ * Git 实现
  *
  * @author reghao
  * @date 2019-10-12 22:22:00
@@ -204,20 +204,4 @@ public class GitImpl implements CodeUpdater {
         int lastIndex = gitUrl.lastIndexOf("/");
         return gitUrl.substring(lastIndex).split(".git")[0];
     }
-
-    public static void main(String[] args) throws Exception {
-        String prikeyFile = "/home/reghao/id_rsa_codeup";
-        GitImpl git = new GitImpl(prikeyFile);
-        String iqbms = "git@codeup.aliyun.com:5f1f8daf6207a1a8b17f6742/FrontEnd/IQuizoo.BMS.git";
-        String dnkt = "git@codeup.aliyun.com:5f1f8daf6207a1a8b17f6742/FrontEnd/IQuizoo.Game.git";
-        String spiderlab = "git@git.reghao.cn:reghao/spiderlab.git";
-
-        String local = "/home/reghao/Downloads/codeup/spiderlab";
-        String branch = "master";
-        CommitLog commitLog = git.update(spiderlab, branch, local);
-        System.out.println();
-
-        /*String spiderlab = "http://git.reghao.cn/reghao/spiderlab.git";
-        GitImpl git1 = new GitImpl("reghao", "gjs");*/
-    }
 }

dmaster/src/main/java/cn/reghao/autodop/dmaster/app/service/tools/updater/SvnImpl.java

@@ -12,7 +12,7 @@ import java.util.List;
 import java.util.Map;
 
 /**
- * svn 实现
+ * SVN 实现
  *
  * @author reghao
  * @date 2019-09-07 16:18:46

dmaster/src/main/java/cn/reghao/autodop/dmaster/common/config/WebMvcConfig.java

@@ -1,5 +1,6 @@
 package cn.reghao.autodop.dmaster.common.config;
 
+import cn.reghao.autodop.dmaster.common.interceptor.WebhookInterceptor;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.http.converter.HttpMessageConverter;
@@ -18,6 +19,12 @@ import java.util.List;
 @Configuration
 @EnableWebMvc
 public class WebMvcConfig implements WebMvcConfigurer {
+    private WebhookInterceptor webhookInterceptor;
+
+    public WebMvcConfig(WebhookInterceptor webhookInterceptor) {
+        this.webhookInterceptor = webhookInterceptor;
+    }
+
     /**
      * swagger-ui 接口页面
      *
@@ -80,4 +87,10 @@ public class WebMvcConfig implements WebMvcConfigurer {
                 .allowCredentials(true)
                 .maxAge(3600);
     }
+
+    @Override
+    public void addInterceptors(InterceptorRegistry registry) {
+        registry.addInterceptor(webhookInterceptor)
+                .addPathPatterns("/api/hook/**");
+    }
 }

dmaster/src/main/java/cn/reghao/autodop/dmaster/common/interceptor/WebhookInterceptor.java

@@ -0,0 +1,58 @@
+package cn.reghao.autodop.dmaster.common.interceptor;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.lang.Nullable;
+import org.springframework.stereotype.Component;
+import org.springframework.web.servlet.HandlerInterceptor;
+import org.springframework.web.servlet.ModelAndView;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+/**
+ * @author reghao
+ * @date 2020-09-08 17:16:37
+ */
+@Slf4j
+@Component
+public class WebhookInterceptor implements HandlerInterceptor {
+    @Override
+    public boolean preHandle(HttpServletRequest request,
+                             HttpServletResponse response,
+                             Object handler) throws Exception {
+        String source = request.getParameter("source");
+        switch (WebhookSource.valueOf(source)) {
+            case codeup:
+                String event = request.getHeader("X-Codeup-Event");
+                String deliveryId = request.getHeader("x-codeup-delivery");
+                if (event == null || deliveryId == null) {
+                    return false;
+                }
+                break;
+            case gogs:
+                break;
+            case git:
+                break;
+            case svn:
+                break;
+            default:
+                return false;
+        }
+        return true;
+    }
+
+    @Override
+    public void postHandle(HttpServletRequest request,
+                           HttpServletResponse response,
+                           Object handler,
+                           @Nullable ModelAndView modelAndView) throws Exception {
+        log.info("postHandle");
+    }
+
+    @Override
+    public void afterCompletion(HttpServletRequest request,
+                                HttpServletResponse response,
+                                Object handler, @Nullable Exception ex) throws Exception {
+        log.info("afterCompletion");
+    }
+}

dmaster/src/main/java/cn/reghao/autodop/dmaster/common/interceptor/WebhookSource.java

@@ -0,0 +1,21 @@
+package cn.reghao.autodop.dmaster.common.interceptor;
+
+/**
+ * 调用 webhook 的源
+ *
+ * @author reghao
+ * @date 2019-10-18 14:31:29
+ */
+public enum WebhookSource {
+    codeup("Codeup"), gogs("Gogs"), git("Git"), svn("SVN");
+
+    private String value;
+
+    WebhookSource(String value) {
+        this.value = value;
+    }
+
+    public String getValue() {
+        return value;
+    }
+}

dmaster/src/test/java/cn/reghao/autodop/dmaster/app/service/tools/updater/GitImplTest.java

@@ -6,11 +6,18 @@ class GitImplTest {
 
     @Test
     void main() throws Exception {
+        String prikeyFile = "/home/reghao/id_rsa_codeup";
+        GitImpl git = new GitImpl(prikeyFile);
+        String iqbms = "git@codeup.aliyun.com:5f1f8daf6207a1a8b17f6742/FrontEnd/IQuizoo.BMS.git";
+        String dnkt = "git@codeup.aliyun.com:5f1f8daf6207a1a8b17f6742/FrontEnd/IQuizoo.Game.git";
+        String spiderlab = "git@git.reghao.cn:reghao/spiderlab.git";
+
+        String local = "/home/reghao/Downloads/codeup/spiderlab";
         String branch = "master";
-        GitImpl git = new GitImpl("");
-        String remote = "http://git.reghao.cn/reghao/spiderlab.git";
-        String local = "/home/reghao/tmp/autodop/opt/data/git/spiderlab";
-        CommitLog commitLog = git.update(remote, branch, local);
-        //System.out.println(git.localVersion(local));
+        CommitLog commitLog = git.update(spiderlab, branch, local);
+        System.out.println();
+
+        /*String spiderlab = "http://git.reghao.cn/reghao/spiderlab.git";
+        GitImpl git1 = new GitImpl("reghao", "gjs");*/
     }
 }