|
|
@@ -1,35 +1,26 @@
|
|
|
package cn.reghao.autodop.dmaster.app.service.build;
|
|
|
|
|
|
-import cn.reghao.autodop.common.notification.DingNotify;
|
|
|
-import cn.reghao.autodop.common.notification.Notify;
|
|
|
import cn.reghao.autodop.dmaster.app.cache.OrchestrationCache;
|
|
|
-import cn.reghao.autodop.dmaster.app.constant.NotifierType;
|
|
|
-import cn.reghao.autodop.dmaster.app.entity.Notification;
|
|
|
-import cn.reghao.autodop.dmaster.app.entity.build.AppBuild;
|
|
|
import cn.reghao.autodop.dmaster.app.entity.build.BuildResult;
|
|
|
+import cn.reghao.autodop.dmaster.app.entity.build.BuildStage;
|
|
|
+import cn.reghao.autodop.dmaster.app.entity.deploy.DeployResult;
|
|
|
import cn.reghao.autodop.dmaster.app.entity.log.BuildLog;
|
|
|
-import cn.reghao.autodop.dmaster.app.entity.log.DeployLog;
|
|
|
-import cn.reghao.autodop.dmaster.app.repository.log.BuildLogRepository;
|
|
|
-import cn.reghao.autodop.dmaster.app.repository.log.CommitLogRepository;
|
|
|
-import cn.reghao.autodop.dmaster.app.repository.log.DeployLogRepository;
|
|
|
import cn.reghao.autodop.dmaster.app.service.deploy.AppDeployer;
|
|
|
-import cn.reghao.autodop.dmaster.app.service.deploy.NotifyTask;
|
|
|
import cn.reghao.autodop.dmaster.app.service.build.tools.updater.CommitLog;
|
|
|
-import cn.reghao.autodop.dmaster.app.vo.log.BuildDeployResult;
|
|
|
import cn.reghao.autodop.dmaster.app.entity.orchestration.AppOrchestration;
|
|
|
import cn.reghao.autodop.dmaster.app.repository.log.LogRepository;
|
|
|
import cn.reghao.autodop.dmaster.app.service.log.LogConsumer;
|
|
|
-import cn.reghao.autodop.dmaster.common.thread.ThreadPoolWrapper;
|
|
|
-import lombok.Data;
|
|
|
+import cn.reghao.autodop.dmaster.common.exception.ExceptionUtil;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
|
|
|
+import java.time.LocalDateTime;
|
|
|
import java.util.*;
|
|
|
import java.util.concurrent.*;
|
|
|
-import java.util.stream.Collectors;
|
|
|
|
|
|
/**
|
|
|
* 应用构建部署分发器
|
|
|
+ * 多个线程访问同一个 BuildDispatcher 对象
|
|
|
*
|
|
|
* @author reghao
|
|
|
* @date 2019-11-12 17:20:27
|
|
|
@@ -37,168 +28,117 @@ import java.util.stream.Collectors;
|
|
|
@Slf4j
|
|
|
@Service
|
|
|
public class BuildDispatcher {
|
|
|
- private Map<String, AppIntegrate> appIntegrateMap;
|
|
|
- private CommitLogRepository commitLogRepository;
|
|
|
- private BuildLogRepository buildLogRepository;
|
|
|
- private DeployLogRepository deployLogRepository;
|
|
|
- private AppDeployer appDeployer;
|
|
|
// TODO 多线程访问的集合
|
|
|
- private Set<String> onBuilding;
|
|
|
- private OrchestrationCache caching;
|
|
|
+ private Set<String> onBuilding = new ConcurrentSkipListSet<>();
|
|
|
+ private Set<String> onDeploying = new ConcurrentSkipListSet<>();
|
|
|
+ private Map<String, AppIntegrate> integrateMap = new ConcurrentHashMap<>();
|
|
|
+ private OrchestrationCache cache;
|
|
|
+ private AppDeployer appDeployer;
|
|
|
private LogConsumer logConsumer;
|
|
|
- private ExecutorService threadPool;
|
|
|
-
|
|
|
- public BuildDispatcher(AppDeployer appDeployer,
|
|
|
- CommitLogRepository commitLogRepository,
|
|
|
- BuildLogRepository buildLogRepository,
|
|
|
- DeployLogRepository deployLogRepository,
|
|
|
- OrchestrationCache caching,
|
|
|
- LogRepository logDAO) {
|
|
|
- this.commitLogRepository = commitLogRepository;
|
|
|
- this.buildLogRepository = buildLogRepository;
|
|
|
- this.deployLogRepository = deployLogRepository;
|
|
|
+
|
|
|
+ public BuildDispatcher(OrchestrationCache orchestrationCache, AppDeployer appDeployer, LogRepository logDAO) {
|
|
|
+ this.cache = orchestrationCache;
|
|
|
this.appDeployer = appDeployer;
|
|
|
- this.onBuilding = new HashSet<>();
|
|
|
- this.appIntegrateMap = new HashMap<>();
|
|
|
- this.caching = caching;
|
|
|
this.logConsumer = new LogConsumer(logDAO);
|
|
|
- this.threadPool = ThreadPoolWrapper.threadPool("build");
|
|
|
- threadPool.submit(logConsumer);
|
|
|
}
|
|
|
|
|
|
- public BuildResult build(String appId) {
|
|
|
- AppIntegrate appIntegrate = appIntegrateMap.get(appId);
|
|
|
- if (appIntegrate != null) {
|
|
|
+ /**
|
|
|
+ * 构建部署应用
|
|
|
+ *
|
|
|
+ * @param
|
|
|
+ * @return
|
|
|
+ * @date 2021-02-06 上午1:46
|
|
|
+ */
|
|
|
+ public BuildResult buildAndDeploy(String appId, boolean isDeploy) {
|
|
|
+ if (!onBuilding.add(appId)) {
|
|
|
+ // TODO 应用正在构建中...
|
|
|
+ return null;
|
|
|
+ }
|
|
|
|
|
|
+ AppIntegrate appIntegrate = integrateMap.get(appId);
|
|
|
+ if (appIntegrate == null) {
|
|
|
+ AppOrchestration app = cache.findByAppId(appId);
|
|
|
+ appIntegrate = new AppIntegrate(app);
|
|
|
+ integrateMap.put(appId, appIntegrate);
|
|
|
}
|
|
|
|
|
|
- return null;
|
|
|
- }
|
|
|
+ BuildLog buildLog = buildTask(appIntegrate);
|
|
|
+ onBuilding.remove(appId);
|
|
|
|
|
|
- public BuildDeployResult buildAndDeploy(String appId, boolean isDeploy) {
|
|
|
- AppOrchestration app = caching.findByAppId(appId);
|
|
|
- return dispatch(app, isDeploy);
|
|
|
- }
|
|
|
+ if (isDeploy) {
|
|
|
+ deploy(buildLog);
|
|
|
+ }
|
|
|
|
|
|
- public BuildDeployResult buildAndDeploy(String repo, String branch, boolean isDeploy) {
|
|
|
- AppOrchestration app = caching.findByRepoAndBranch(repo, branch);
|
|
|
- return dispatch(app, isDeploy);
|
|
|
+ return null;
|
|
|
}
|
|
|
|
|
|
- public BuildDeployResult dispatch(AppOrchestration app, boolean isDeploy) {
|
|
|
- String appId = app.getAppId();
|
|
|
- if (!onBuilding.contains(appId)) {
|
|
|
- onBuilding.add(appId);
|
|
|
- } else {
|
|
|
- log.info("{} 正在构建中...", appId);
|
|
|
- }
|
|
|
+ /**
|
|
|
+ * 更新 -> 编译 -> 打包 -> 部署流水线
|
|
|
+ *
|
|
|
+ * @date 2019-11-07 下午10:14
|
|
|
+ */
|
|
|
+ private BuildLog buildTask(AppIntegrate appIntegrate) {
|
|
|
+ BuildLog buildLog = new BuildLog();
|
|
|
+ buildLog.setAppId(appIntegrate.appId());
|
|
|
|
|
|
- AppIntegrate appIntegrate = getAppIntegrate(app);
|
|
|
- Future<BuildDeployResult> future = threadPool.submit(new BuildDeployPipeline(appIntegrate, isDeploy, logConsumer));
|
|
|
- while (!future.isDone() && !future.isCancelled()) {
|
|
|
- try {
|
|
|
- Thread.sleep(1_000);
|
|
|
- } catch (InterruptedException e) {
|
|
|
- e.printStackTrace();
|
|
|
- }
|
|
|
+ long start = System.currentTimeMillis();
|
|
|
+ try {
|
|
|
+ CommitLog commitLog = appIntegrate.update();
|
|
|
+ buildLog.setUpdate(commitLog.isUpdate());
|
|
|
+ buildLog.setCommitId(commitLog.getCommitId());
|
|
|
+ buildLog.setUpdateTotalTime(System.currentTimeMillis()-start);
|
|
|
+ logConsumer.addCommitLog(commitLog);
|
|
|
+ } catch (Exception e) {
|
|
|
+ buildException(buildLog, BuildStage.update, e);
|
|
|
+ return buildLog;
|
|
|
}
|
|
|
|
|
|
- BuildDeployResult buildDeployResult = null;
|
|
|
try {
|
|
|
- buildDeployResult = future.get();
|
|
|
- if (buildDeployResult.isDeploy()) {
|
|
|
- // TODO 异步发送通知
|
|
|
- log.info("异步发送通知");
|
|
|
- notifyAsync(app, buildDeployResult);
|
|
|
- }
|
|
|
- } catch (InterruptedException | ExecutionException e) {
|
|
|
- e.printStackTrace();
|
|
|
- } finally {
|
|
|
- onBuilding.remove(appId);
|
|
|
+ start = System.currentTimeMillis();
|
|
|
+ appIntegrate.compile();
|
|
|
+ buildLog.setCompileTotalTime(System.currentTimeMillis()-start);
|
|
|
+ } catch (Exception e) {
|
|
|
+ buildException(buildLog, BuildStage.compile, e);
|
|
|
+ return buildLog;
|
|
|
}
|
|
|
- return buildDeployResult;
|
|
|
- }
|
|
|
|
|
|
- public BuildDeployResult deploy(String appId, String commitId) {
|
|
|
- AppOrchestration app = caching.findByAppId(appId);
|
|
|
- CommitLog commitLog = commitLogRepository.findByCommitId(commitId);
|
|
|
- AppBuild appBuild = app.getAppBuild();
|
|
|
- //BuildLog buildLog = buildLogRepository.findByIsDeleteFalseAndCommitLogAndAppCompileAndAppPackAndAppIdAndStatusCode(commitLog, appBuild.getAppCompile(), appBuild.getAppPack(), appId, 0);
|
|
|
- BuildLog buildLog = new BuildLog();
|
|
|
- if (buildLog != null) {
|
|
|
- //Set<String> hosts = app.getAppDeploy().getHosts();
|
|
|
- Set<String> hosts = new HashSet<>();
|
|
|
- // 当前 commit 最近一次部署时已部署的主机
|
|
|
- Set<String> deployedHosts = deployLogRepository.findByIsDeleteFalseAndBuildLog(buildLog).stream()
|
|
|
- .filter(deployLog -> deployLog.getStatusCode() != 0)
|
|
|
- .map(DeployLog::getHost)
|
|
|
- .collect(Collectors.toSet());
|
|
|
-
|
|
|
- // TODO 比较 toDeployHosts 和 hosts,查看是否有新添加但未部署的主机
|
|
|
- if (!hosts.isEmpty()) {
|
|
|
- List<DeployLog> deployLogs1 = getAppIntegrate(app).deploy(buildLog, hosts);
|
|
|
- logConsumer.addDeployLogs(deployLogs1);
|
|
|
- BuildDeployResult buildDeployResult = BuildDeployResult.fromDeployLogs(deployLogs1);
|
|
|
- notifyAsync(app, buildDeployResult);
|
|
|
- return buildDeployResult;
|
|
|
- } else {
|
|
|
- BuildDeployResult buildDeployResult = new BuildDeployResult();
|
|
|
- buildDeployResult.setAppId(appId);
|
|
|
- buildDeployResult.setDeploy(true);
|
|
|
- buildDeployResult.setCommitId(commitId);
|
|
|
- buildDeployResult.setStatusCode(0);
|
|
|
- buildDeployResult.setMsg("当前版本已部署到所有节点");
|
|
|
- return buildDeployResult;
|
|
|
- }
|
|
|
- } else {
|
|
|
- BuildDeployResult buildDeployResult = new BuildDeployResult();
|
|
|
- buildDeployResult.setAppId(appId);
|
|
|
- buildDeployResult.setDeploy(true);
|
|
|
- buildDeployResult.setCommitId(commitId);
|
|
|
- buildDeployResult.setStatusCode(1);
|
|
|
- buildDeployResult.setMsg(appId + " 或 " + commitId + "不存在");
|
|
|
- return buildDeployResult;
|
|
|
+ try {
|
|
|
+ start = System.currentTimeMillis();
|
|
|
+ String packagePath = appIntegrate.pack();
|
|
|
+ buildLog.setPackagePath(packagePath);
|
|
|
+ buildLog.setPackTotalTime(System.currentTimeMillis()-start);
|
|
|
+ } catch (Exception e) {
|
|
|
+ buildException(buildLog, BuildStage.pack, e);
|
|
|
+ return buildLog;
|
|
|
}
|
|
|
- }
|
|
|
|
|
|
- private AppIntegrate getAppIntegrate(AppOrchestration app) {
|
|
|
- return new AppIntegrate(app, appDeployer);
|
|
|
- /*AppIntegrate appIntegrate = map.get(app.getAppId());
|
|
|
- if (appIntegrate == null) {
|
|
|
- appIntegrate = new AppIntegrate(app, appDeployer);
|
|
|
- // TODO AppOrchestration 更新后 AppIntegrate 需要重新获取
|
|
|
- map.put(app.getAppId(), appIntegrate);
|
|
|
- }
|
|
|
- return appIntegrate;*/
|
|
|
+ buildLog.setStage(BuildStage.done);
|
|
|
+ buildLog.setStatusCode(0);
|
|
|
+ buildLog.setBuildTime(LocalDateTime.now());
|
|
|
+ return buildLog;
|
|
|
}
|
|
|
|
|
|
- public void notifyAsync(AppOrchestration app, BuildDeployResult buildDeployResult) {
|
|
|
- Notification notification = app.getNotification();
|
|
|
- String notifierType = notification.getNotifierType();
|
|
|
- String dest = notification.getUrl();
|
|
|
- Notify notify = null;
|
|
|
- if (notifierType.equals(NotifierType.webhook.name())) {
|
|
|
- notify = new DingNotify();
|
|
|
- }
|
|
|
-
|
|
|
- // TODO 检查应用是否成功运行,若启动失败,则回退到上一个版本。若上一个版本仍失败,则返回错误并发出紧急通知
|
|
|
- NotifyMsg notifyMsg = new NotifyMsg(app.getAppId(), app.getEnv(), buildDeployResult.getCommitId());
|
|
|
- notifyMsg.setMsg(buildDeployResult.getMsg());
|
|
|
- // TODO 检测通知是否成功发送
|
|
|
- threadPool.submit(new NotifyTask(notify, dest, notifyMsg));
|
|
|
+ private void buildException(BuildLog log, BuildStage stage, Exception e) {
|
|
|
+ log.setStage(stage);
|
|
|
+ log.setStatusCode(1);
|
|
|
+ log.setErrDetail(ExceptionUtil.errorMsg(e));
|
|
|
+ log.setBuildTime(LocalDateTime.now());
|
|
|
}
|
|
|
|
|
|
- @Data
|
|
|
- static class NotifyMsg {
|
|
|
- private String appId;
|
|
|
- private String env;
|
|
|
- private String commitId;
|
|
|
- private String msg;
|
|
|
-
|
|
|
- public NotifyMsg(String appId, String env, String commitId) {
|
|
|
- this.appId = appId;
|
|
|
- this.env = env;
|
|
|
- this.commitId = commitId;
|
|
|
+ /**
|
|
|
+ * 部署应用的最新版本
|
|
|
+ *
|
|
|
+ * @param
|
|
|
+ * @return
|
|
|
+ * @date 2021-02-06 上午2:02
|
|
|
+ */
|
|
|
+ public DeployResult deploy(BuildLog buildLog) {
|
|
|
+ if (!onDeploying.add(buildLog.getAppId())) {
|
|
|
+ // TODO 应用正在部署中...
|
|
|
+ return null;
|
|
|
}
|
|
|
+
|
|
|
+ onDeploying.remove(buildLog.getAppId());
|
|
|
+ return null;
|
|
|
}
|
|
|
}
|