From ba2a8e9d2e76a0048211d522410b6935ab3c781f Mon Sep 17 00:00:00 2001 From: Zhiguo Wu Date: Wed, 1 Jan 2025 11:39:17 +0800 Subject: [PATCH] BIGTOP-4314: Adjust host add command (#137) --- .../apache/bigtop/manager/dao/po/JobPO.java | 5 -- .../apache/bigtop/manager/dao/po/StagePO.java | 5 -- .../factory/host/HostAddJobFactory.java | 49 ++++++++++++ .../command/job/AbstractServiceJob.java | 3 +- .../server/command/job/ClusterAddJob.java | 10 ++- .../server/command/job/HostAddJob.java | 80 +++++++++++++++++++ .../server/command/stage/StageContext.java | 37 ++++++--- .../command/validator/HostAddValidator.java | 3 +- .../server/model/converter/HostConverter.java | 5 ++ .../model/dto/command/ClusterCommandDTO.java | 4 +- .../model/dto/command/HostCommandDTO.java | 25 +++++- .../model/req/command/ClusterCommandReq.java | 3 +- .../model/req/command/HostCommandReq.java | 40 +++++++++- .../manager/server/model/vo/StageVO.java | 4 - 14 files changed, 233 insertions(+), 40 deletions(-) create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/factory/host/HostAddJobFactory.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/HostAddJob.java diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/JobPO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/JobPO.java index d787c3fff..c0c8bfb9b 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/JobPO.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/JobPO.java @@ -20,14 +20,12 @@ import lombok.Data; import lombok.EqualsAndHashCode; -import lombok.ToString; import jakarta.persistence.Column; import jakarta.persistence.Id; import jakarta.persistence.Lob; import jakarta.persistence.Table; import java.io.Serializable; -import java.util.List; @Data @EqualsAndHashCode(callSuper = true) @@ -50,7 +48,4 @@ public class JobPO extends BasePO implements Serializable { @Column(name = "cluster_id") private Long clusterId; - - @ToString.Exclude - private List stages; } diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/StagePO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/StagePO.java index b2f8cdaf9..078316553 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/StagePO.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/StagePO.java @@ -20,13 +20,11 @@ import lombok.Data; import lombok.EqualsAndHashCode; -import lombok.ToString; import jakarta.persistence.Column; import jakarta.persistence.Id; import jakarta.persistence.Table; import java.io.Serializable; -import java.util.List; @Data @EqualsAndHashCode(callSuper = true) @@ -60,7 +58,4 @@ public class StagePO extends BasePO implements Serializable { @Column(name = "job_id", nullable = false) private Long jobId; - - @ToString.Exclude - private List tasks; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/factory/host/HostAddJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/factory/host/HostAddJobFactory.java new file mode 100644 index 000000000..ecf9b9117 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/factory/host/HostAddJobFactory.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.server.command.factory.host; + +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.factory.cluster.AbstractClusterJobFactory; +import org.apache.bigtop.manager.server.command.job.HostAddJob; +import org.apache.bigtop.manager.server.command.job.Job; +import org.apache.bigtop.manager.server.command.job.JobContext; +import org.apache.bigtop.manager.server.enums.CommandLevel; + +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class HostAddJobFactory extends AbstractClusterJobFactory { + + @Override + public CommandIdentifier getCommandIdentifier() { + return new CommandIdentifier(CommandLevel.HOST, Command.ADD); + } + + @Override + public Job createJob(JobContext jobContext) { + return new HostAddJob(jobContext); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java index 996cb3c5e..8f128f1a0 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java @@ -20,7 +20,6 @@ import org.apache.bigtop.manager.common.constants.ComponentCategories; import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.common.utils.JsonUtils; import org.apache.bigtop.manager.dao.po.ComponentPO; import org.apache.bigtop.manager.dao.query.ComponentQuery; import org.apache.bigtop.manager.dao.repository.ComponentDao; @@ -147,7 +146,7 @@ protected List findHostnamesByComponentName(String componentName) { } protected void createCacheStage() { - StageContext stageContext = StageContext.fromPayload(JsonUtils.writeAsString(jobContext.getCommandDTO())); + StageContext stageContext = StageContext.fromCommandDTO(jobContext.getCommandDTO()); stages.add(new CacheFileUpdateStage(stageContext)); } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ClusterAddJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ClusterAddJob.java index 95fe9e116..6bf52feee 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ClusterAddJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ClusterAddJob.java @@ -35,6 +35,8 @@ import org.apache.bigtop.manager.server.model.dto.HostDTO; import org.apache.bigtop.manager.server.service.HostService; +import java.util.List; + public class ClusterAddJob extends AbstractJob { private HostService hostService; @@ -92,9 +94,11 @@ private void saveCluster() { private void saveHosts() { CommandDTO commandDTO = jobContext.getCommandDTO(); - HostDTO hostDTO = commandDTO.getClusterCommand().getHosts(); - hostDTO.setClusterId(clusterPO.getId()); - hostService.add(hostDTO); + List hostDTOList = commandDTO.getClusterCommand().getHosts(); + for (HostDTO hostDTO : hostDTOList) { + hostDTO.setClusterId(clusterPO.getId()); + hostService.add(hostDTO); + } } private void linkJobToCluster() { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/HostAddJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/HostAddJob.java new file mode 100644 index 000000000..6f2d9b1aa --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/HostAddJob.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.server.command.job; + +import org.apache.bigtop.manager.server.command.stage.CacheFileUpdateStage; +import org.apache.bigtop.manager.server.command.stage.HostCheckStage; +import org.apache.bigtop.manager.server.command.stage.SetupJdkStage; +import org.apache.bigtop.manager.server.command.stage.StageContext; +import org.apache.bigtop.manager.server.holder.SpringContextHolder; +import org.apache.bigtop.manager.server.model.converter.HostConverter; +import org.apache.bigtop.manager.server.model.dto.CommandDTO; +import org.apache.bigtop.manager.server.model.dto.HostDTO; +import org.apache.bigtop.manager.server.service.HostService; + +import java.util.List; + +public class HostAddJob extends AbstractJob { + + private HostService hostService; + + public HostAddJob(JobContext jobContext) { + super(jobContext); + } + + @Override + protected void injectBeans() { + super.injectBeans(); + + hostService = SpringContextHolder.getBean(HostService.class); + } + + @Override + protected void createStages() { + StageContext stageContext = StageContext.fromCommandDTO(jobContext.getCommandDTO()); + stages.add(new HostCheckStage(stageContext)); + stages.add(new CacheFileUpdateStage(stageContext)); + stages.add(new SetupJdkStage(stageContext)); + } + + @Override + public void beforeRun() { + super.beforeRun(); + + if (jobContext.getRetryFlag()) { + return; + } + + saveHosts(); + } + + @Override + public String getName() { + return "Add hosts"; + } + + private void saveHosts() { + CommandDTO commandDTO = jobContext.getCommandDTO(); + List hostDTOList = HostConverter.INSTANCE.fromCommand2DTO(commandDTO.getHostCommands()); + for (HostDTO hostDTO : hostDTOList) { + hostDTO.setClusterId(clusterPO.getId()); + hostService.add(hostDTO); + } + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/StageContext.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/StageContext.java index cc1536cee..a3823b4c6 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/StageContext.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/StageContext.java @@ -18,7 +18,9 @@ */ package org.apache.bigtop.manager.server.command.stage; -import org.apache.bigtop.manager.common.utils.JsonUtils; +import org.apache.bigtop.manager.dao.po.ClusterPO; +import org.apache.bigtop.manager.dao.repository.ClusterDao; +import org.apache.bigtop.manager.server.holder.SpringContextHolder; import org.apache.bigtop.manager.server.model.dto.CommandDTO; import org.apache.bigtop.manager.server.model.dto.ComponentDTO; import org.apache.bigtop.manager.server.model.dto.ServiceDTO; @@ -45,14 +47,17 @@ public class StageContext { private ComponentDTO componentDTO; - public static StageContext fromPayload(String payload) { - CommandDTO commandDTO = JsonUtils.readFromString(payload, CommandDTO.class); - return fromCommandDTO(commandDTO); - } - public static StageContext fromCommandDTO(CommandDTO commandDTO) { StageContext context = new StageContext(); - context.setClusterId(commandDTO.getClusterId()); + + if (commandDTO.getClusterId() != null) { + ClusterDao clusterDao = SpringContextHolder.getBean(ClusterDao.class); + ClusterPO clusterPO = clusterDao.findById(commandDTO.getClusterId()); + context.setClusterId(clusterPO.getId()); + context.setClusterName(clusterPO.getName()); + context.setUserGroup(clusterPO.getUserGroup()); + context.setRootDir(clusterPO.getRootDir()); + } switch (commandDTO.getCommandLevel()) { case CLUSTER -> fromClusterCommandPayload(context, commandDTO); @@ -67,14 +72,22 @@ public static StageContext fromCommandDTO(CommandDTO commandDTO) { private static void fromClusterCommandPayload(StageContext context, CommandDTO commandDTO) { ClusterCommandDTO clusterCommand = commandDTO.getClusterCommand(); - context.setClusterName(clusterCommand.getName()); - context.setHostnames(clusterCommand.getHosts().getHostnames()); - context.setUserGroup(clusterCommand.getUserGroup()); - context.setRootDir(clusterCommand.getRootDir()); + if (context.getClusterId() == null) { + List hostnames = clusterCommand.getHosts().stream() + .flatMap(hostDTO -> hostDTO.getHostnames().stream()) + .toList(); + context.setHostnames(hostnames); + context.setClusterName(clusterCommand.getName()); + context.setUserGroup(clusterCommand.getUserGroup()); + context.setRootDir(clusterCommand.getRootDir()); + } } private static void fromHostCommandPayload(StageContext context, CommandDTO commandDTO) { - // No need to set anything here, we should deal with this in the host job factory + List hostnames = commandDTO.getHostCommands().stream() + .flatMap(hostDTO -> hostDTO.getHostnames().stream()) + .toList(); + context.setHostnames(hostnames); } private static void fromServiceCommandPayload(StageContext context, CommandDTO commandDTO) { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/validator/HostAddValidator.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/validator/HostAddValidator.java index ae20aa7d5..d10e6bac2 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/validator/HostAddValidator.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/validator/HostAddValidator.java @@ -25,7 +25,6 @@ import org.apache.bigtop.manager.server.enums.ApiExceptionEnum; import org.apache.bigtop.manager.server.enums.CommandLevel; import org.apache.bigtop.manager.server.exception.ApiException; -import org.apache.bigtop.manager.server.model.dto.command.HostCommandDTO; import org.apache.commons.collections4.CollectionUtils; @@ -48,7 +47,7 @@ public List getCommandIdentifiers() { @Override public void validate(ValidatorContext context) { List hostnames = context.getCommandDTO().getHostCommands().stream() - .map(HostCommandDTO::getHostname) + .flatMap(hostCommandDTO -> hostCommandDTO.getHostnames().stream()) .toList(); List hostPOList = hostDao.findAllByHostnames(hostnames); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/HostConverter.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/HostConverter.java index 0b02ce230..a0ff5e0d0 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/HostConverter.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/HostConverter.java @@ -21,6 +21,7 @@ import org.apache.bigtop.manager.dao.po.HostPO; import org.apache.bigtop.manager.server.config.MapStructSharedConfig; import org.apache.bigtop.manager.server.model.dto.HostDTO; +import org.apache.bigtop.manager.server.model.dto.command.HostCommandDTO; import org.apache.bigtop.manager.server.model.req.HostReq; import org.apache.bigtop.manager.server.model.vo.HostVO; @@ -37,6 +38,10 @@ public interface HostConverter { HostDTO fromReq2DTO(HostReq hostReq); + HostDTO fromCommand2DTO(HostCommandDTO hostCommandDTO); + + List fromCommand2DTO(List hostCommandDTOList); + HostDTO fromPO2DTO(HostPO hostPO); List fromPO2DTO(List hostPOList); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/command/ClusterCommandDTO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/command/ClusterCommandDTO.java index 25f400078..a9f32a7e2 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/command/ClusterCommandDTO.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/command/ClusterCommandDTO.java @@ -22,6 +22,8 @@ import lombok.Data; +import java.util.List; + @Data public class ClusterCommandDTO { @@ -37,5 +39,5 @@ public class ClusterCommandDTO { private String rootDir; - private HostDTO hosts; + private List hosts; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/command/HostCommandDTO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/command/HostCommandDTO.java index 947648802..4c01776bf 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/command/HostCommandDTO.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/command/HostCommandDTO.java @@ -21,9 +21,32 @@ import lombok.Data; import java.io.Serializable; +import java.util.List; @Data public class HostCommandDTO implements Serializable { - private String hostname; + private List hostnames; + + private String agentDir; + + private Long clusterId; + + private String sshUser; + + private Integer sshPort; + + private Integer authType; + + private String sshPassword; + + private String sshKeyString; + + private String sshKeyFilename; + + private String sshKeyPassword; + + private Integer grpcPort; + + private String desc; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/command/ClusterCommandReq.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/command/ClusterCommandReq.java index 1d5ae3bab..e75d71b12 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/command/ClusterCommandReq.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/command/ClusterCommandReq.java @@ -26,6 +26,7 @@ import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; +import java.util.List; @Data public class ClusterCommandReq { @@ -52,5 +53,5 @@ public class ClusterCommandReq { @NotEmpty(groups = {CommandGroupSequenceProvider.ClusterCommandGroup.class}) @Schema(description = "Hosts info for this cluster") - private HostReq hosts; + private List hosts; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/command/HostCommandReq.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/command/HostCommandReq.java index d7b8f6533..f6ba2ee43 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/command/HostCommandReq.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/command/HostCommandReq.java @@ -21,12 +21,44 @@ import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; -import jakarta.validation.constraints.NotBlank; +import java.util.List; @Data public class HostCommandReq { - @NotBlank - @Schema(description = "Host Name", example = "bigtop-manager-server") - private String hostname; + @Schema(example = "[host1, host2]") + private List hostnames; + + @Schema(example = "/opt") + private String agentDir; + + @Schema(example = "1") + private Long clusterId; + + @Schema(example = "root") + private String sshUser; + + @Schema(example = "22") + private Integer sshPort; + + @Schema(example = "1") + private Integer authType; + + @Schema(example = "password") + private String sshPassword; + + @Schema(example = "sshKeyString") + private String sshKeyString; + + @Schema(example = "id_rsa123") + private String sshKeyFilename; + + @Schema(example = "password") + private String sshKeyPassword; + + @Schema(example = "8835") + private Integer grpcPort; + + @Schema(example = "description") + private String desc; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/StageVO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/StageVO.java index f47429f43..e5ace5af2 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/StageVO.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/StageVO.java @@ -20,8 +20,6 @@ import lombok.Data; -import java.util.List; - @Data public class StageVO { @@ -29,8 +27,6 @@ public class StageVO { private String name; - private List tasks; - private String state; private Integer order;