BIGTOP-4314: Adjust host add command (apache#137)

bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/JobPO.java

@@ -20,14 +20,12 @@
 
 import lombok.Data;
 import lombok.EqualsAndHashCode;
-import lombok.ToString;
 
 import jakarta.persistence.Column;
 import jakarta.persistence.Id;
 import jakarta.persistence.Lob;
 import jakarta.persistence.Table;
 import java.io.Serializable;
-import java.util.List;
 
 @Data
 @EqualsAndHashCode(callSuper = true)
@@ -50,7 +48,4 @@ public class JobPO extends BasePO implements Serializable {
 
     @Column(name = "cluster_id")
     private Long clusterId;
-
-    @ToString.Exclude
-    private List<StagePO> stages;
 }

bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/StagePO.java

@@ -20,13 +20,11 @@
 
 import lombok.Data;
 import lombok.EqualsAndHashCode;
-import lombok.ToString;
 
 import jakarta.persistence.Column;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
 import java.io.Serializable;
-import java.util.List;
 
 @Data
 @EqualsAndHashCode(callSuper = true)
@@ -60,7 +58,4 @@ public class StagePO extends BasePO implements Serializable {
 
     @Column(name = "job_id", nullable = false)
     private Long jobId;
-
-    @ToString.Exclude
-    private List<TaskPO> tasks;
 }

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/factory/host/HostAddJobFactory.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.server.command.factory.host;
+
+import org.apache.bigtop.manager.common.enums.Command;
+import org.apache.bigtop.manager.server.command.CommandIdentifier;
+import org.apache.bigtop.manager.server.command.factory.cluster.AbstractClusterJobFactory;
+import org.apache.bigtop.manager.server.command.job.HostAddJob;
+import org.apache.bigtop.manager.server.command.job.Job;
+import org.apache.bigtop.manager.server.command.job.JobContext;
+import org.apache.bigtop.manager.server.enums.CommandLevel;
+
+import org.springframework.beans.factory.config.ConfigurableBeanFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Component
+@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
+public class HostAddJobFactory extends AbstractClusterJobFactory {
+
+    @Override
+    public CommandIdentifier getCommandIdentifier() {
+        return new CommandIdentifier(CommandLevel.HOST, Command.ADD);
+    }
+
+    @Override
+    public Job createJob(JobContext jobContext) {
+        return new HostAddJob(jobContext);
+    }
+}

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java

@@ -20,7 +20,6 @@
 
 import org.apache.bigtop.manager.common.constants.ComponentCategories;
 import org.apache.bigtop.manager.common.enums.Command;
-import org.apache.bigtop.manager.common.utils.JsonUtils;
 import org.apache.bigtop.manager.dao.po.ComponentPO;
 import org.apache.bigtop.manager.dao.query.ComponentQuery;
 import org.apache.bigtop.manager.dao.repository.ComponentDao;
@@ -147,7 +146,7 @@ protected List<String> findHostnamesByComponentName(String componentName) {
     }
 
     protected void createCacheStage() {
-        StageContext stageContext = StageContext.fromPayload(JsonUtils.writeAsString(jobContext.getCommandDTO()));
+        StageContext stageContext = StageContext.fromCommandDTO(jobContext.getCommandDTO());
         stages.add(new CacheFileUpdateStage(stageContext));
     }
 

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ClusterAddJob.java

@@ -35,6 +35,8 @@
 import org.apache.bigtop.manager.server.model.dto.HostDTO;
 import org.apache.bigtop.manager.server.service.HostService;
 
+import java.util.List;
+
 public class ClusterAddJob extends AbstractJob {
 
     private HostService hostService;
@@ -92,9 +94,11 @@ private void saveCluster() {
 
     private void saveHosts() {
         CommandDTO commandDTO = jobContext.getCommandDTO();
-        HostDTO hostDTO = commandDTO.getClusterCommand().getHosts();
-        hostDTO.setClusterId(clusterPO.getId());
-        hostService.add(hostDTO);
+        List<HostDTO> hostDTOList = commandDTO.getClusterCommand().getHosts();
+        for (HostDTO hostDTO : hostDTOList) {
+            hostDTO.setClusterId(clusterPO.getId());
+            hostService.add(hostDTO);
+        }
     }
 
     private void linkJobToCluster() {

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/HostAddJob.java

@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.server.command.job;
+
+import org.apache.bigtop.manager.server.command.stage.CacheFileUpdateStage;
+import org.apache.bigtop.manager.server.command.stage.HostCheckStage;
+import org.apache.bigtop.manager.server.command.stage.SetupJdkStage;
+import org.apache.bigtop.manager.server.command.stage.StageContext;
+import org.apache.bigtop.manager.server.holder.SpringContextHolder;
+import org.apache.bigtop.manager.server.model.converter.HostConverter;
+import org.apache.bigtop.manager.server.model.dto.CommandDTO;
+import org.apache.bigtop.manager.server.model.dto.HostDTO;
+import org.apache.bigtop.manager.server.service.HostService;
+
+import java.util.List;
+
+public class HostAddJob extends AbstractJob {
+
+    private HostService hostService;
+
+    public HostAddJob(JobContext jobContext) {
+        super(jobContext);
+    }
+
+    @Override
+    protected void injectBeans() {
+        super.injectBeans();
+
+        hostService = SpringContextHolder.getBean(HostService.class);
+    }
+
+    @Override
+    protected void createStages() {
+        StageContext stageContext = StageContext.fromCommandDTO(jobContext.getCommandDTO());
+        stages.add(new HostCheckStage(stageContext));
+        stages.add(new CacheFileUpdateStage(stageContext));
+        stages.add(new SetupJdkStage(stageContext));
+    }
+
+    @Override
+    public void beforeRun() {
+        super.beforeRun();
+
+        if (jobContext.getRetryFlag()) {
+            return;
+        }
+
+        saveHosts();
+    }
+
+    @Override
+    public String getName() {
+        return "Add hosts";
+    }
+
+    private void saveHosts() {
+        CommandDTO commandDTO = jobContext.getCommandDTO();
+        List<HostDTO> hostDTOList = HostConverter.INSTANCE.fromCommand2DTO(commandDTO.getHostCommands());
+        for (HostDTO hostDTO : hostDTOList) {
+            hostDTO.setClusterId(clusterPO.getId());
+            hostService.add(hostDTO);
+        }
+    }
+}

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/StageContext.java

@@ -18,7 +18,9 @@
  */
 package org.apache.bigtop.manager.server.command.stage;
 
-import org.apache.bigtop.manager.common.utils.JsonUtils;
+import org.apache.bigtop.manager.dao.po.ClusterPO;
+import org.apache.bigtop.manager.dao.repository.ClusterDao;
+import org.apache.bigtop.manager.server.holder.SpringContextHolder;
 import org.apache.bigtop.manager.server.model.dto.CommandDTO;
 import org.apache.bigtop.manager.server.model.dto.ComponentDTO;
 import org.apache.bigtop.manager.server.model.dto.ServiceDTO;
@@ -45,14 +47,17 @@ public class StageContext {
 
     private ComponentDTO componentDTO;
 
-    public static StageContext fromPayload(String payload) {
-        CommandDTO commandDTO = JsonUtils.readFromString(payload, CommandDTO.class);
-        return fromCommandDTO(commandDTO);
-    }
-
     public static StageContext fromCommandDTO(CommandDTO commandDTO) {
         StageContext context = new StageContext();
-        context.setClusterId(commandDTO.getClusterId());
+
+        if (commandDTO.getClusterId() != null) {
+            ClusterDao clusterDao = SpringContextHolder.getBean(ClusterDao.class);
+            ClusterPO clusterPO = clusterDao.findById(commandDTO.getClusterId());
+            context.setClusterId(clusterPO.getId());
+            context.setClusterName(clusterPO.getName());
+            context.setUserGroup(clusterPO.getUserGroup());
+            context.setRootDir(clusterPO.getRootDir());
+        }
 
         switch (commandDTO.getCommandLevel()) {
             case CLUSTER -> fromClusterCommandPayload(context, commandDTO);
@@ -67,14 +72,22 @@ public static StageContext fromCommandDTO(CommandDTO commandDTO) {
     private static void fromClusterCommandPayload(StageContext context, CommandDTO commandDTO) {
         ClusterCommandDTO clusterCommand = commandDTO.getClusterCommand();
 
-        context.setClusterName(clusterCommand.getName());
-        context.setHostnames(clusterCommand.getHosts().getHostnames());
-        context.setUserGroup(clusterCommand.getUserGroup());
-        context.setRootDir(clusterCommand.getRootDir());
+        if (context.getClusterId() == null) {
+            List<String> hostnames = clusterCommand.getHosts().stream()
+                    .flatMap(hostDTO -> hostDTO.getHostnames().stream())
+                    .toList();
+            context.setHostnames(hostnames);
+            context.setClusterName(clusterCommand.getName());
+            context.setUserGroup(clusterCommand.getUserGroup());
+            context.setRootDir(clusterCommand.getRootDir());
+        }
     }
 
     private static void fromHostCommandPayload(StageContext context, CommandDTO commandDTO) {
-        // No need to set anything here, we should deal with this in the host job factory
+        List<String> hostnames = commandDTO.getHostCommands().stream()
+                .flatMap(hostDTO -> hostDTO.getHostnames().stream())
+                .toList();
+        context.setHostnames(hostnames);
     }
 
     private static void fromServiceCommandPayload(StageContext context, CommandDTO commandDTO) {

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/validator/HostAddValidator.java

@@ -25,7 +25,6 @@
 import org.apache.bigtop.manager.server.enums.ApiExceptionEnum;
 import org.apache.bigtop.manager.server.enums.CommandLevel;
 import org.apache.bigtop.manager.server.exception.ApiException;
-import org.apache.bigtop.manager.server.model.dto.command.HostCommandDTO;
 
 import org.apache.commons.collections4.CollectionUtils;
 
@@ -48,7 +47,7 @@ public List<CommandIdentifier> getCommandIdentifiers() {
     @Override
     public void validate(ValidatorContext context) {
         List<String> hostnames = context.getCommandDTO().getHostCommands().stream()
-                .map(HostCommandDTO::getHostname)
+                .flatMap(hostCommandDTO -> hostCommandDTO.getHostnames().stream())
                 .toList();
 
         List<HostPO> hostPOList = hostDao.findAllByHostnames(hostnames);

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/HostConverter.java

@@ -21,6 +21,7 @@
 import org.apache.bigtop.manager.dao.po.HostPO;
 import org.apache.bigtop.manager.server.config.MapStructSharedConfig;
 import org.apache.bigtop.manager.server.model.dto.HostDTO;
+import org.apache.bigtop.manager.server.model.dto.command.HostCommandDTO;
 import org.apache.bigtop.manager.server.model.req.HostReq;
 import org.apache.bigtop.manager.server.model.vo.HostVO;
 
@@ -37,6 +38,10 @@ public interface HostConverter {
 
     HostDTO fromReq2DTO(HostReq hostReq);
 
+    HostDTO fromCommand2DTO(HostCommandDTO hostCommandDTO);
+
+    List<HostDTO> fromCommand2DTO(List<HostCommandDTO> hostCommandDTOList);
+
     HostDTO fromPO2DTO(HostPO hostPO);
 
     List<HostDTO> fromPO2DTO(List<HostPO> hostPOList);

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/command/ClusterCommandDTO.java

@@ -22,6 +22,8 @@
 
 import lombok.Data;
 
+import java.util.List;
+
 @Data
 public class ClusterCommandDTO {
 
@@ -37,5 +39,5 @@ public class ClusterCommandDTO {
 
     private String rootDir;
 
-    private HostDTO hosts;
+    private List<HostDTO> hosts;
 }

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/command/HostCommandDTO.java

@@ -21,9 +21,32 @@
 import lombok.Data;
 
 import java.io.Serializable;
+import java.util.List;
 
 @Data
 public class HostCommandDTO implements Serializable {
 
-    private String hostname;
+    private List<String> hostnames;
+
+    private String agentDir;
+
+    private Long clusterId;
+
+    private String sshUser;
+
+    private Integer sshPort;
+
+    private Integer authType;
+
+    private String sshPassword;
+
+    private String sshKeyString;
+
+    private String sshKeyFilename;
+
+    private String sshKeyPassword;
+
+    private Integer grpcPort;
+
+    private String desc;
 }

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/command/ClusterCommandReq.java

@@ -26,6 +26,7 @@
 
 import jakarta.validation.constraints.NotEmpty;
 import jakarta.validation.constraints.NotNull;
+import java.util.List;
 
 @Data
 public class ClusterCommandReq {
@@ -52,5 +53,5 @@ public class ClusterCommandReq {
 
     @NotEmpty(groups = {CommandGroupSequenceProvider.ClusterCommandGroup.class})
     @Schema(description = "Hosts info for this cluster")
-    private HostReq hosts;
+    private List<HostReq> hosts;
 }