diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/HostPO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/HostPO.java index 908495584..9d685ab0c 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/HostPO.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/HostPO.java @@ -39,6 +39,9 @@ public class HostPO extends BasePO implements Serializable { @Column(name = "hostname") private String hostname; + @Column(name = "agent_dir") + private String agentDir; + @Column(name = "ssh_user") private String sshUser; diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ServicePO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ServicePO.java index d7b220033..0b9fffce8 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ServicePO.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ServicePO.java @@ -53,8 +53,8 @@ public class ServicePO extends BasePO implements Serializable { @Column(name = "stack") private String stack; - @Column(name = "need_restart") - private Boolean needRestart; + @Column(name = "restart_flag") + private Boolean restartFlag; @Column(name = "cluster_id") private Long clusterId; diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/query/ServiceQuery.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/query/ServiceQuery.java index 46e38df7e..86830c534 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/query/ServiceQuery.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/query/ServiceQuery.java @@ -33,7 +33,7 @@ public class ServiceQuery { private Long clusterId; - private Boolean needRestart; + private Boolean restartFlag; private Integer status; } diff --git a/bigtop-manager-dao/src/main/resources/mapper/mysql/HostMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/mysql/HostMapper.xml index 17bd3826a..9a16c413c 100644 --- a/bigtop-manager-dao/src/main/resources/mapper/mysql/HostMapper.xml +++ b/bigtop-manager-dao/src/main/resources/mapper/mysql/HostMapper.xml @@ -24,11 +24,13 @@ - id, hostname, ipv4, ipv6, os, arch, available_processors, free_memory_size, total_memory_size, free_disk, total_disk, status, cluster_id + id, hostname, agent_dir, ssh_user, ssh_port, auth_type, ssh_password, ssh_key_string, ssh_key_filename, ssh_key_password, + grpc_port, ipv4, ipv6, os, arch, available_processors, free_memory_size, total_memory_size, + free_disk, total_disk, desc, status, err_info, cluster_id - ${alias}.id, ${alias}.hostname, ${alias}.ssh_user, ${alias}.ssh_port, ${alias}.auth_type, + ${alias}.id, ${alias}.hostname, ${alias}.agent_dir, ${alias}.ssh_user, ${alias}.ssh_port, ${alias}.auth_type, ${alias}.ssh_password, ${alias}.ssh_key_string, ${alias}.ssh_key_filename, ${alias}.ssh_key_password, ${alias}.grpc_port, ${alias}.ipv4, ${alias}.ipv6, ${alias}.os, ${alias}.arch, ${alias}.available_processors, ${alias}.free_memory_size, ${alias}.total_memory_size, diff --git a/bigtop-manager-dao/src/main/resources/mapper/mysql/ServiceMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/mysql/ServiceMapper.xml index 4d1f03930..f1179956c 100644 --- a/bigtop-manager-dao/src/main/resources/mapper/mysql/ServiceMapper.xml +++ b/bigtop-manager-dao/src/main/resources/mapper/mysql/ServiceMapper.xml @@ -24,11 +24,11 @@ - id, name, display_name, `desc`, user, version, stack, need_restart, cluster_id, status + id, name, display_name, `desc`, user, version, stack, restart_flag, cluster_id, status - ${alias}.id, ${alias}.name, ${alias}.display_name, ${alias}.`desc`, ${alias}.user, ${alias}.version, ${alias}.stack, ${alias}.need_restart, ${alias}.cluster_id, ${alias}.status + ${alias}.id, ${alias}.name, ${alias}.display_name, ${alias}.`desc`, ${alias}.user, ${alias}.version, ${alias}.stack, ${alias}.restart_flag, ${alias}.cluster_id, ${alias}.status @@ -43,8 +43,8 @@ and s.cluster_id = #{query.clusterId} - - and s.need_restart = #{query.needRestart} + + and s.restart_flag = #{query.restartFlag} and s.status = #{query.status} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java index a57655708..996cb3c5e 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java @@ -76,13 +76,13 @@ protected void beforeCreateStages() { super.beforeCreateStages(); } - protected StageContext createStageContext(String serviceName, String componentName, List hostIds) { + protected StageContext createStageContext(String serviceName, String componentName, List hostnames) { StageContext stageContext = StageContext.fromCommandDTO(jobContext.getCommandDTO()); ServiceDTO serviceDTO = StackUtils.getServiceDTO(serviceName); ComponentDTO componentDTO = StackUtils.getComponentDTO(componentName); - stageContext.setHostIds(hostIds); + stageContext.setHostnames(hostnames); stageContext.setServiceDTO(serviceDTO); stageContext.setComponentDTO(componentDTO); @@ -133,7 +133,7 @@ protected Boolean isClientComponent(String componentName) { return componentDTO.getCategory().equalsIgnoreCase(ComponentCategories.CLIENT); } - protected List findHostIdsByComponentName(String componentName) { + protected List findHostnamesByComponentName(String componentName) { ComponentQuery componentQuery = ComponentQuery.builder() .clusterId(clusterPO.getId()) .name(componentName) @@ -142,7 +142,7 @@ protected List findHostIdsByComponentName(String componentName) { if (componentPOList == null) { return new ArrayList<>(); } else { - return componentPOList.stream().map(ComponentPO::getHostId).toList(); + return componentPOList.stream().map(ComponentPO::getHostname).toList(); } } @@ -158,12 +158,12 @@ protected void createAddStages() { String[] split = componentCommand.split("-"); String componentName = split[0]; String serviceName = findServiceNameByComponentName(componentName); - List hostIds = findHostIdsByComponentName(componentName); - if (CollectionUtils.isEmpty(hostIds)) { + List hostnames = findHostnamesByComponentName(componentName); + if (CollectionUtils.isEmpty(hostnames)) { continue; } - StageContext stageContext = createStageContext(serviceName, componentName, hostIds); + StageContext stageContext = createStageContext(serviceName, componentName, hostnames); stages.add(new ComponentAddStage(stageContext)); } } @@ -173,9 +173,9 @@ protected void createConfigureStages() { for (ComponentHostDTO componentHost : serviceCommand.getComponentHosts()) { String serviceName = serviceCommand.getServiceName(); String componentName = componentHost.getComponentName(); - List hostIds = componentHost.getHostIds(); + List hostnames = componentHost.getHostnames(); - StageContext stageContext = createStageContext(serviceName, componentName, hostIds); + StageContext stageContext = createStageContext(serviceName, componentName, hostnames); stages.add(new ComponentConfigureStage(stageContext)); } } @@ -193,12 +193,12 @@ protected void createStartStages() { continue; } - List hostIds = findHostIdsByComponentName(componentName); - if (CollectionUtils.isEmpty(hostIds)) { + List hostnames = findHostnamesByComponentName(componentName); + if (CollectionUtils.isEmpty(hostnames)) { continue; } - StageContext stageContext = createStageContext(serviceName, componentName, hostIds); + StageContext stageContext = createStageContext(serviceName, componentName, hostnames); stages.add(new ComponentStartStage(stageContext)); } } @@ -215,12 +215,12 @@ protected void createStopStages() { continue; } - List hostIds = findHostIdsByComponentName(componentName); - if (CollectionUtils.isEmpty(hostIds)) { + List hostnames = findHostnamesByComponentName(componentName); + if (CollectionUtils.isEmpty(hostnames)) { continue; } - StageContext stageContext = createStageContext(serviceName, componentName, hostIds); + StageContext stageContext = createStageContext(serviceName, componentName, hostnames); stages.add(new ComponentStopStage(stageContext)); } } @@ -237,12 +237,12 @@ protected void createCheckStages() { continue; } - List hostIds = findHostIdsByComponentName(componentName); - if (CollectionUtils.isEmpty(hostIds)) { + List hostnames = findHostnamesByComponentName(componentName); + if (CollectionUtils.isEmpty(hostnames)) { continue; } - StageContext stageContext = createStageContext(serviceName, componentName, List.of(hostIds.get(0))); + StageContext stageContext = createStageContext(serviceName, componentName, List.of(hostnames.get(0))); stages.add(new ComponentCheckStage(stageContext)); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ClusterAddJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ClusterAddJob.java index cad25f8f5..95fe9e116 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ClusterAddJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ClusterAddJob.java @@ -18,11 +18,9 @@ */ package org.apache.bigtop.manager.server.command.job; -import org.apache.bigtop.manager.dao.po.HostPO; import org.apache.bigtop.manager.dao.po.JobPO; import org.apache.bigtop.manager.dao.po.StagePO; import org.apache.bigtop.manager.dao.po.TaskPO; -import org.apache.bigtop.manager.dao.repository.HostDao; import org.apache.bigtop.manager.server.command.stage.CacheFileUpdateStage; import org.apache.bigtop.manager.server.command.stage.HostCheckStage; import org.apache.bigtop.manager.server.command.stage.SetupJdkStage; @@ -34,13 +32,12 @@ import org.apache.bigtop.manager.server.model.converter.ClusterConverter; import org.apache.bigtop.manager.server.model.dto.ClusterDTO; import org.apache.bigtop.manager.server.model.dto.CommandDTO; - -import java.util.ArrayList; -import java.util.List; +import org.apache.bigtop.manager.server.model.dto.HostDTO; +import org.apache.bigtop.manager.server.service.HostService; public class ClusterAddJob extends AbstractJob { - private HostDao hostDao; + private HostService hostService; public ClusterAddJob(JobContext jobContext) { super(jobContext); @@ -50,7 +47,7 @@ public ClusterAddJob(JobContext jobContext) { protected void injectBeans() { super.injectBeans(); - hostDao = SpringContextHolder.getBean(HostDao.class); + hostService = SpringContextHolder.getBean(HostService.class); } @Override @@ -64,15 +61,14 @@ protected void createStages() { @Override public void beforeRun() { super.beforeRun(); - } - @Override - public void onSuccess() { - super.onSuccess(); + if (jobContext.getRetryFlag()) { + return; + } saveCluster(); - linkHostToCluster(); + saveHosts(); linkJobToCluster(); } @@ -94,18 +90,11 @@ private void saveCluster() { clusterDao.save(clusterPO); } - private void linkHostToCluster() { + private void saveHosts() { CommandDTO commandDTO = jobContext.getCommandDTO(); - List ids = commandDTO.getClusterCommand().getHostIds(); - List hostPOList = new ArrayList<>(); - for (Long id : ids) { - HostPO hostPO = new HostPO(); - hostPO.setId(id); - hostPO.setClusterId(clusterPO.getId()); - hostPOList.add(hostPO); - } - - hostDao.partialUpdateByIds(hostPOList); + HostDTO hostDTO = commandDTO.getClusterCommand().getHosts(); + hostDTO.setClusterId(clusterPO.getId()); + hostService.add(hostDTO); } private void linkJobToCluster() { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceAddJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceAddJob.java index 3c708bcaf..9498ccd35 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceAddJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceAddJob.java @@ -86,12 +86,12 @@ protected List getComponentNames() { } @Override - protected List findHostIdsByComponentName(String componentName) { + protected List findHostnamesByComponentName(String componentName) { for (ServiceCommandDTO serviceCommand : jobContext.getCommandDTO().getServiceCommands()) { List componentHosts = serviceCommand.getComponentHosts(); for (ComponentHostDTO componentHost : componentHosts) { if (componentHost.getComponentName().equals(componentName)) { - return componentHost.getHostIds(); + return componentHost.getHostnames(); } } } @@ -156,7 +156,7 @@ private void saveService(ServiceCommandDTO serviceCommand) { List componentPOList = new ArrayList<>(); for (ComponentHostDTO componentHostDTO : serviceCommand.getComponentHosts()) { String componentName = componentHostDTO.getComponentName(); - List hostPOList = hostDao.findByIds(componentHostDTO.getHostIds()); + List hostPOList = hostDao.findAllByHostnames(componentHostDTO.getHostnames()); for (HostPO hostPO : hostPOList) { ComponentDTO componentDTO = StackUtils.getComponentDTO(componentName); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractComponentStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractComponentStage.java index 04f2d6cbf..29c86a8d6 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractComponentStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractComponentStage.java @@ -23,7 +23,6 @@ import org.apache.bigtop.manager.server.command.task.TaskContext; import org.apache.bigtop.manager.server.holder.SpringContextHolder; import org.apache.bigtop.manager.server.model.dto.ComponentDTO; -import org.apache.bigtop.manager.server.model.dto.HostDTO; import org.apache.bigtop.manager.server.model.dto.ServiceDTO; import java.util.HashMap; @@ -61,12 +60,12 @@ protected String getComponentName() { return stageContext.getComponentDTO().getName(); } - protected TaskContext createTaskContext(HostDTO hostDTO) { + protected TaskContext createTaskContext(String hostname) { ServiceDTO serviceDTO = stageContext.getServiceDTO(); ComponentDTO componentDTO = stageContext.getComponentDTO(); TaskContext taskContext = new TaskContext(); - taskContext.setHostDTO(hostDTO); + taskContext.setHostname(hostname); taskContext.setClusterId(clusterPO.getId()); taskContext.setClusterName(clusterPO.getName()); taskContext.setServiceName(serviceDTO.getName()); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractStage.java index 9dfaf0035..47f99871d 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractStage.java @@ -25,8 +25,6 @@ import org.apache.bigtop.manager.dao.repository.StageDao; import org.apache.bigtop.manager.server.command.task.Task; import org.apache.bigtop.manager.server.holder.SpringContextHolder; -import org.apache.bigtop.manager.server.model.converter.HostConverter; -import org.apache.bigtop.manager.server.model.dto.HostDTO; import lombok.extern.slf4j.Slf4j; @@ -56,9 +54,8 @@ public AbstractStage(StageContext stageContext) { beforeCreateTasks(); - List hostDTOList = HostConverter.INSTANCE.fromPO2DTO(hostDao.findByIds(stageContext.getHostIds())); - for (HostDTO hostDTO : hostDTOList) { - tasks.add(createTask(hostDTO)); + for (String hostname : stageContext.getHostnames()) { + tasks.add(createTask(hostname)); } } @@ -69,7 +66,7 @@ protected void injectBeans() { protected abstract void beforeCreateTasks(); - protected abstract Task createTask(HostDTO hostDTO); + protected abstract Task createTask(String hostname); protected String getServiceName() { return "cluster"; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/CacheFileUpdateStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/CacheFileUpdateStage.java index 0d25c0a83..88c57081c 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/CacheFileUpdateStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/CacheFileUpdateStage.java @@ -22,7 +22,6 @@ import org.apache.bigtop.manager.server.command.task.CacheFileUpdateTask; import org.apache.bigtop.manager.server.command.task.Task; import org.apache.bigtop.manager.server.command.task.TaskContext; -import org.apache.bigtop.manager.server.model.dto.HostDTO; import java.util.ArrayList; import java.util.HashMap; @@ -42,24 +41,24 @@ protected void injectBeans() { @Override protected void beforeCreateTasks() { - List hostIds = new ArrayList<>(); + List hostnames = new ArrayList<>(); if (stageContext.getClusterId() == null) { - hostIds.addAll(stageContext.getHostIds() == null ? List.of() : stageContext.getHostIds()); + hostnames.addAll(stageContext.getHostnames() == null ? List.of() : stageContext.getHostnames()); } else { - hostIds.addAll(stageContext.getHostIds() == null ? List.of() : stageContext.getHostIds()); - hostIds.addAll(hostDao.findAllByClusterId(stageContext.getClusterId()).stream() - .map(HostPO::getId) + hostnames.addAll(stageContext.getHostnames() == null ? List.of() : stageContext.getHostnames()); + hostnames.addAll(hostDao.findAllByClusterId(stageContext.getClusterId()).stream() + .map(HostPO::getHostname) .toList()); } - stageContext.setHostIds(hostIds); + stageContext.setHostnames(hostnames); } @Override - protected Task createTask(HostDTO hostDTO) { + protected Task createTask(String hostname) { TaskContext taskContext = new TaskContext(); - taskContext.setHostDTO(hostDTO); + taskContext.setHostname(hostname); taskContext.setClusterId(stageContext.getClusterId()); taskContext.setClusterName(stageContext.getClusterName()); taskContext.setUserGroup(stageContext.getUserGroup()); @@ -70,7 +69,7 @@ protected Task createTask(HostDTO hostDTO) { taskContext.setComponentDisplayName("Agent"); Map properties = new HashMap<>(); - properties.put("hostIds", stageContext.getHostIds()); + properties.put("hostnames", stageContext.getHostnames()); taskContext.setProperties(properties); return new CacheFileUpdateTask(taskContext); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentAddStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentAddStage.java index 6afefea18..883ce8ef4 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentAddStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentAddStage.java @@ -20,7 +20,6 @@ import org.apache.bigtop.manager.server.command.task.ComponentAddTask; import org.apache.bigtop.manager.server.command.task.Task; -import org.apache.bigtop.manager.server.model.dto.HostDTO; public class ComponentAddStage extends AbstractComponentStage { @@ -29,8 +28,8 @@ public ComponentAddStage(StageContext stageContext) { } @Override - protected Task createTask(HostDTO hostDTO) { - return new ComponentAddTask(createTaskContext(hostDTO)); + protected Task createTask(String hostname) { + return new ComponentAddTask(createTaskContext(hostname)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentCheckStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentCheckStage.java index 759702614..7498ce7d6 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentCheckStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentCheckStage.java @@ -20,7 +20,6 @@ import org.apache.bigtop.manager.server.command.task.ComponentCheckTask; import org.apache.bigtop.manager.server.command.task.Task; -import org.apache.bigtop.manager.server.model.dto.HostDTO; public class ComponentCheckStage extends AbstractComponentStage { @@ -29,8 +28,8 @@ public ComponentCheckStage(StageContext stageContext) { } @Override - protected Task createTask(HostDTO hostDTO) { - return new ComponentCheckTask(createTaskContext(hostDTO)); + protected Task createTask(String hostname) { + return new ComponentCheckTask(createTaskContext(hostname)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentConfigureStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentConfigureStage.java index d1aa4519f..08eea2ad5 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentConfigureStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentConfigureStage.java @@ -20,7 +20,6 @@ import org.apache.bigtop.manager.server.command.task.ComponentConfigureTask; import org.apache.bigtop.manager.server.command.task.Task; -import org.apache.bigtop.manager.server.model.dto.HostDTO; public class ComponentConfigureStage extends AbstractComponentStage { @@ -29,8 +28,8 @@ public ComponentConfigureStage(StageContext stageContext) { } @Override - protected Task createTask(HostDTO hostDTO) { - return new ComponentConfigureTask(createTaskContext(hostDTO)); + protected Task createTask(String hostname) { + return new ComponentConfigureTask(createTaskContext(hostname)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentStartStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentStartStage.java index fffef4832..00f67f8a7 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentStartStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentStartStage.java @@ -20,7 +20,6 @@ import org.apache.bigtop.manager.server.command.task.ComponentStartTask; import org.apache.bigtop.manager.server.command.task.Task; -import org.apache.bigtop.manager.server.model.dto.HostDTO; public class ComponentStartStage extends AbstractComponentStage { @@ -29,8 +28,8 @@ public ComponentStartStage(StageContext stageContext) { } @Override - protected Task createTask(HostDTO hostDTO) { - return new ComponentStartTask(createTaskContext(hostDTO)); + protected Task createTask(String hostname) { + return new ComponentStartTask(createTaskContext(hostname)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentStopStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentStopStage.java index 8133f106f..b36781506 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentStopStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentStopStage.java @@ -20,7 +20,6 @@ import org.apache.bigtop.manager.server.command.task.ComponentStopTask; import org.apache.bigtop.manager.server.command.task.Task; -import org.apache.bigtop.manager.server.model.dto.HostDTO; public class ComponentStopStage extends AbstractComponentStage { @@ -29,8 +28,8 @@ public ComponentStopStage(StageContext stageContext) { } @Override - protected Task createTask(HostDTO hostDTO) { - return new ComponentStopTask(createTaskContext(hostDTO)); + protected Task createTask(String hostname) { + return new ComponentStopTask(createTaskContext(hostname)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/HostCheckStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/HostCheckStage.java index 49ca5f62a..5c9c1b658 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/HostCheckStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/HostCheckStage.java @@ -21,7 +21,6 @@ import org.apache.bigtop.manager.server.command.task.HostCheckTask; import org.apache.bigtop.manager.server.command.task.Task; import org.apache.bigtop.manager.server.command.task.TaskContext; -import org.apache.bigtop.manager.server.model.dto.HostDTO; public class HostCheckStage extends AbstractStage { @@ -38,9 +37,9 @@ protected void injectBeans() { protected void beforeCreateTasks() {} @Override - protected Task createTask(HostDTO hostDTO) { + protected Task createTask(String hostname) { TaskContext taskContext = new TaskContext(); - taskContext.setHostDTO(hostDTO); + taskContext.setHostname(hostname); taskContext.setClusterId(stageContext.getClusterId()); taskContext.setClusterName(stageContext.getClusterName()); taskContext.setServiceName("cluster"); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/SetupJdkStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/SetupJdkStage.java index 023a5bd91..d3f0d1612 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/SetupJdkStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/SetupJdkStage.java @@ -21,7 +21,6 @@ import org.apache.bigtop.manager.server.command.task.SetupJdkTask; import org.apache.bigtop.manager.server.command.task.Task; import org.apache.bigtop.manager.server.command.task.TaskContext; -import org.apache.bigtop.manager.server.model.dto.HostDTO; public class SetupJdkStage extends AbstractStage { @@ -38,9 +37,9 @@ protected void injectBeans() { protected void beforeCreateTasks() {} @Override - protected Task createTask(HostDTO hostDTO) { + protected Task createTask(String hostname) { TaskContext taskContext = new TaskContext(); - taskContext.setHostDTO(hostDTO); + taskContext.setHostname(hostname); taskContext.setClusterId(stageContext.getClusterId()); taskContext.setClusterName(stageContext.getClusterName()); taskContext.setServiceName("cluster"); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/StageContext.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/StageContext.java index ec6995061..cc1536cee 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/StageContext.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/StageContext.java @@ -39,7 +39,7 @@ public class StageContext { private String rootDir; - private List hostIds; + private List hostnames; private ServiceDTO serviceDTO; @@ -68,7 +68,7 @@ private static void fromClusterCommandPayload(StageContext context, CommandDTO c ClusterCommandDTO clusterCommand = commandDTO.getClusterCommand(); context.setClusterName(clusterCommand.getName()); - context.setHostIds(clusterCommand.getHostIds()); + context.setHostnames(clusterCommand.getHosts().getHostnames()); context.setUserGroup(clusterCommand.getUserGroup()); context.setRootDir(clusterCommand.getRootDir()); } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractTask.java index becfd6ab7..0d1ed195e 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractTask.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractTask.java @@ -23,6 +23,7 @@ import org.apache.bigtop.manager.common.enums.JobState; import org.apache.bigtop.manager.common.utils.JsonUtils; import org.apache.bigtop.manager.dao.po.TaskPO; +import org.apache.bigtop.manager.dao.repository.HostDao; import org.apache.bigtop.manager.dao.repository.TaskDao; import org.apache.bigtop.manager.grpc.generated.CommandReply; import org.apache.bigtop.manager.grpc.generated.CommandRequest; @@ -30,6 +31,7 @@ import org.apache.bigtop.manager.grpc.utils.ProtobufUtil; import org.apache.bigtop.manager.server.grpc.GrpcClient; import org.apache.bigtop.manager.server.holder.SpringContextHolder; +import org.apache.bigtop.manager.server.model.converter.HostConverter; import org.apache.bigtop.manager.server.model.dto.HostDTO; import lombok.extern.slf4j.Slf4j; @@ -37,6 +39,8 @@ @Slf4j public abstract class AbstractTask implements Task { + protected HostDao hostDao; + protected TaskDao taskDao; protected TaskContext taskContext; @@ -55,6 +59,7 @@ public AbstractTask(TaskContext taskContext) { } protected void injectBeans() { + this.hostDao = SpringContextHolder.getBean(HostDao.class); this.taskDao = SpringContextHolder.getBean(TaskDao.class); } @@ -83,7 +88,7 @@ public Boolean run() { builder.setTaskId(getTaskPO().getId()); commandRequest = builder.build(); - HostDTO hostDTO = taskContext.getHostDTO(); + HostDTO hostDTO = HostConverter.INSTANCE.fromPO2DTO(hostDao.findByHostname(taskContext.getHostname())); CommandServiceGrpc.CommandServiceBlockingStub stub = GrpcClient.getBlockingStub( hostDTO.getHostname(), hostDTO.getGrpcPort(), CommandServiceGrpc.CommandServiceBlockingStub.class); CommandReply reply = stub.exec(commandRequest); @@ -135,7 +140,7 @@ public TaskPO getTaskPO() { taskPO = new TaskPO(); taskPO.setName(getName()); taskPO.setContext(JsonUtils.writeAsString(taskContext)); - taskPO.setHostname(taskContext.getHostDTO().getHostname()); + taskPO.setHostname(taskContext.getHostname()); taskPO.setServiceName(taskContext.getServiceName()); taskPO.setServiceUser(taskContext.getServiceUser()); taskPO.setComponentName(taskContext.getComponentName()); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/CacheFileUpdateTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/CacheFileUpdateTask.java index d231a38b6..42468e4ef 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/CacheFileUpdateTask.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/CacheFileUpdateTask.java @@ -101,7 +101,7 @@ private void genCaches() { @SuppressWarnings("unchecked") private void genFullCaches() { Long clusterId = taskContext.getClusterId(); - List hostIds = (List) taskContext.getProperties().get("hostIds"); + List hostnames = (List) taskContext.getProperties().get("hostnames"); ClusterPO clusterPO = clusterDao.findById(clusterId); ComponentQuery componentQuery = @@ -110,7 +110,7 @@ private void genFullCaches() { List serviceConfigPOList = serviceConfigDao.findByClusterId(clusterPO.getId()); List componentPOList = componentDao.findByQuery(componentQuery); List repoPOList = repoDao.findAll(); - List hostPOList = hostDao.findByIds(hostIds); + List hostPOList = hostDao.findAllByHostnames(hostnames); clusterInfo = new ClusterInfo(); clusterInfo.setName(clusterPO.getName()); @@ -165,10 +165,10 @@ private void genFullCaches() { @SuppressWarnings("unchecked") private void genEmptyCaches() { - List hostIds = (List) taskContext.getProperties().get("hostIds"); + List hostnames = (List) taskContext.getProperties().get("hostnames"); List repoPOList = repoDao.findAll(); - List hostPOList = hostDao.findByIds(hostIds); + List hostPOList = hostDao.findAllByHostnames(hostnames); componentInfoMap = new HashMap<>(); serviceConfigMap = new HashMap<>(); @@ -224,6 +224,6 @@ protected CommandRequest getCommandRequest() { @Override public String getName() { - return "Update cache files on " + taskContext.getHostDTO().getHostname(); + return "Update cache files on " + taskContext.getHostname(); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentAddTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentAddTask.java index fc50700ee..4c8ea99e5 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentAddTask.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentAddTask.java @@ -42,7 +42,7 @@ public void onSuccess() { super.onSuccess(); String componentName = taskContext.getComponentName(); - String hostname = taskContext.getHostDTO().getHostname(); + String hostname = taskContext.getHostname(); ComponentQuery componentQuery = ComponentQuery.builder().hostname(hostname).name(componentName).build(); ComponentPO componentPO = componentDao.findByQuery(componentQuery).get(0); @@ -61,7 +61,6 @@ public void onSuccess() { @Override public String getName() { - return "Add " + taskContext.getComponentDisplayName() + " on " - + taskContext.getHostDTO().getHostname(); + return "Add " + taskContext.getComponentDisplayName() + " on " + taskContext.getHostname(); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentCheckTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentCheckTask.java index e387da4c1..121b5b92c 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentCheckTask.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentCheckTask.java @@ -33,7 +33,6 @@ protected Command getCommand() { @Override public String getName() { - return "Check " + taskContext.getComponentDisplayName() + " on " - + taskContext.getHostDTO().getHostname(); + return "Check " + taskContext.getComponentDisplayName() + " on " + taskContext.getHostname(); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentConfigureTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentConfigureTask.java index e18738999..553d35ab8 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentConfigureTask.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentConfigureTask.java @@ -33,7 +33,6 @@ protected Command getCommand() { @Override public String getName() { - return "Configure " + taskContext.getComponentDisplayName() + " on " - + taskContext.getHostDTO().getHostname(); + return "Configure " + taskContext.getComponentDisplayName() + " on " + taskContext.getHostname(); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentStartTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentStartTask.java index a582daf47..451aef25b 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentStartTask.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentStartTask.java @@ -39,7 +39,7 @@ public void onSuccess() { super.onSuccess(); String componentName = taskContext.getComponentName(); - String hostname = taskContext.getHostDTO().getHostname(); + String hostname = taskContext.getHostname(); ComponentQuery componentQuery = ComponentQuery.builder().hostname(hostname).name(componentName).build(); ComponentPO componentPO = componentDao.findByQuery(componentQuery).get(0); @@ -49,7 +49,6 @@ public void onSuccess() { @Override public String getName() { - return "Start " + taskContext.getComponentDisplayName() + " on " - + taskContext.getHostDTO().getHostname(); + return "Start " + taskContext.getComponentDisplayName() + " on " + taskContext.getHostname(); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentStopTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentStopTask.java index 073017999..24d8294e4 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentStopTask.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentStopTask.java @@ -39,7 +39,7 @@ public void onSuccess() { super.onSuccess(); String componentName = taskContext.getComponentName(); - String hostname = taskContext.getHostDTO().getHostname(); + String hostname = taskContext.getHostname(); ComponentQuery componentQuery = ComponentQuery.builder().hostname(hostname).name(componentName).build(); ComponentPO componentPO = componentDao.findByQuery(componentQuery).get(0); @@ -49,7 +49,6 @@ public void onSuccess() { @Override public String getName() { - return "Stop " + taskContext.getComponentDisplayName() + " on " - + taskContext.getHostDTO().getHostname(); + return "Stop " + taskContext.getComponentDisplayName() + " on " + taskContext.getHostname(); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/HostCheckTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/HostCheckTask.java index b37b432e3..94e6296a7 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/HostCheckTask.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/HostCheckTask.java @@ -53,6 +53,6 @@ protected CommandRequest getCommandRequest() { @Override public String getName() { - return "Check host " + taskContext.getHostDTO().getHostname(); + return "Check host " + taskContext.getHostname(); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/SetupJdkTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/SetupJdkTask.java index 179171bd0..bcaf184a3 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/SetupJdkTask.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/SetupJdkTask.java @@ -47,6 +47,6 @@ protected CommandRequest getCommandRequest() { @Override public String getName() { - return "Setup jdk for " + taskContext.getHostDTO().getHostname(); + return "Setup jdk for " + taskContext.getHostname(); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/TaskContext.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/TaskContext.java index 934d0d244..92e99e4d0 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/TaskContext.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/TaskContext.java @@ -18,8 +18,6 @@ */ package org.apache.bigtop.manager.server.command.task; -import org.apache.bigtop.manager.server.model.dto.HostDTO; - import lombok.Data; import java.util.Map; @@ -31,7 +29,7 @@ public class TaskContext { private String clusterName; - private HostDTO hostDTO; + private String hostname; private String serviceName; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/controller/HostController.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/controller/HostController.java index b8fdafcf3..aa53c51a7 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/controller/HostController.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/controller/HostController.java @@ -21,9 +21,9 @@ import org.apache.bigtop.manager.dao.query.HostQuery; import org.apache.bigtop.manager.server.model.converter.HostConverter; import org.apache.bigtop.manager.server.model.dto.HostDTO; -import org.apache.bigtop.manager.server.model.req.HostPathReq; import org.apache.bigtop.manager.server.model.req.HostReq; import org.apache.bigtop.manager.server.model.vo.HostVO; +import org.apache.bigtop.manager.server.model.vo.InstalledStatusVO; import org.apache.bigtop.manager.server.model.vo.PageVO; import org.apache.bigtop.manager.server.service.HostService; import org.apache.bigtop.manager.server.utils.ResponseEntity; @@ -107,7 +107,14 @@ public ResponseEntity checkConnection(@RequestBody @Validated HostReq h @Operation(summary = "Install dependencies", description = "Install dependencies on a host") @PostMapping("/install-dependencies") - public ResponseEntity checkConnection(@RequestBody @Validated HostPathReq hostPathReq) { - return ResponseEntity.success(hostService.installDependencies(hostPathReq.getHostIds(), hostPathReq.getPath())); + public ResponseEntity installDependencies(@RequestBody @Validated HostReq hostReq) { + HostDTO hostDTO = HostConverter.INSTANCE.fromReq2DTO(hostReq); + return ResponseEntity.success(hostService.installDependencies(hostDTO)); + } + + @Operation(summary = "Installed status", description = "Install status for a host") + @GetMapping("/installed-status") + public ResponseEntity> installedStatus() { + return ResponseEntity.success(hostService.installedStatus()); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/enums/InstalledStatusEnum.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/enums/InstalledStatusEnum.java new file mode 100644 index 000000000..543f4856c --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/enums/InstalledStatusEnum.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.server.enums; + +import lombok.Getter; + +@Getter +public enum InstalledStatusEnum { + INSTALLING(1), + SUCCESS(2), + FAILED(3), + ; + + private final Integer code; + + InstalledStatusEnum(Integer code) { + this.code = code; + } + + public static InstalledStatusEnum fromCode(Integer code) { + for (InstalledStatusEnum status : InstalledStatusEnum.values()) { + if (status.code.equals(code)) { + return status; + } + } + + return FAILED; + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/ComponentHostDTO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/ComponentHostDTO.java index 3d6f27aa8..d2b4f8d92 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/ComponentHostDTO.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/ComponentHostDTO.java @@ -30,5 +30,5 @@ public class ComponentHostDTO implements Serializable { private String componentName; @NotEmpty - private List hostIds; + private List hostnames; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/HostDTO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/HostDTO.java index 6e8ff4f11..4581d3acc 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/HostDTO.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/HostDTO.java @@ -33,6 +33,8 @@ public class HostDTO { // Used when converted from PO private String hostname; + private String agentDir; + private Long clusterId; private String sshUser; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/command/ClusterCommandDTO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/command/ClusterCommandDTO.java index 9b97662be..25f400078 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/command/ClusterCommandDTO.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/command/ClusterCommandDTO.java @@ -18,9 +18,9 @@ */ package org.apache.bigtop.manager.server.model.dto.command; -import lombok.Data; +import org.apache.bigtop.manager.server.model.dto.HostDTO; -import java.util.List; +import lombok.Data; @Data public class ClusterCommandDTO { @@ -37,5 +37,5 @@ public class ClusterCommandDTO { private String rootDir; - private List hostIds; + private HostDTO hosts; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/ComponentHostReq.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/ComponentHostReq.java index 1d3b9e862..34ed4d134 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/ComponentHostReq.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/ComponentHostReq.java @@ -32,6 +32,6 @@ public class ComponentHostReq { private String componentName; @NotEmpty - @Schema(description = "Host IDs for component", example = "[1]") - private List hostIds; + @Schema(description = "Hostnames for component", example = "[host1]") + private List hostnames; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/HostReq.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/HostReq.java index 35eb5317a..3adbd95ea 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/HostReq.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/HostReq.java @@ -29,6 +29,9 @@ public class HostReq { @Schema(example = "[host1, host2]") private List hostnames; + @Schema(example = "/opt") + private String agentDir; + @Schema(example = "1") private Long clusterId; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/command/ClusterCommandReq.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/command/ClusterCommandReq.java index 2ee6b8851..1d5ae3bab 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/command/ClusterCommandReq.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/command/ClusterCommandReq.java @@ -18,12 +18,14 @@ */ package org.apache.bigtop.manager.server.model.req.command; +import org.apache.bigtop.manager.server.config.CommandGroupSequenceProvider; +import org.apache.bigtop.manager.server.model.req.HostReq; + import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; -import java.util.List; @Data public class ClusterCommandReq { @@ -48,6 +50,7 @@ public class ClusterCommandReq { @Schema(example = "/opt") private String rootDir; - @Schema(example = "[1, 2]") - private List hostIds; + @NotEmpty(groups = {CommandGroupSequenceProvider.ClusterCommandGroup.class}) + @Schema(description = "Hosts info for this cluster") + private HostReq hosts; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/HostVO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/HostVO.java index e8545b405..f79cab2f2 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/HostVO.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/HostVO.java @@ -27,6 +27,8 @@ public class HostVO { private String hostname; + private String agentDir; + private String sshUser; private Integer sshPort; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/InstalledStatusVO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/InstalledStatusVO.java new file mode 100644 index 000000000..e84e55b4d --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/InstalledStatusVO.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.server.model.vo; + +import org.apache.bigtop.manager.server.enums.InstalledStatusEnum; + +import lombok.Data; + +@Data +public class InstalledStatusVO { + + private String hostname; + + private InstalledStatusEnum status; + + private String message; +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/ServiceVO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/ServiceVO.java index 14df0c084..185532ad7 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/ServiceVO.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/ServiceVO.java @@ -39,7 +39,7 @@ public class ServiceVO { private String stack; - private Boolean needRestart; + private Boolean restartFlag; private Integer status; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/HostService.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/HostService.java index 210cae4bf..96e501764 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/HostService.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/HostService.java @@ -21,6 +21,7 @@ import org.apache.bigtop.manager.dao.query.HostQuery; import org.apache.bigtop.manager.server.model.dto.HostDTO; import org.apache.bigtop.manager.server.model.vo.HostVO; +import org.apache.bigtop.manager.server.model.vo.InstalledStatusVO; import org.apache.bigtop.manager.server.model.vo.PageVO; import java.util.List; @@ -75,9 +76,15 @@ public interface HostService { /** * Install dependencies * - * @param hostIds host ids - * @param path remote host path + * @param hostDTO host infos * @return true if all dependencies are installed */ - Boolean installDependencies(List hostIds, String path); + Boolean installDependencies(HostDTO hostDTO); + + /** + * Get dependency installed status + * + * @return installed status + */ + List installedStatus(); } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java index 70b37eb9e..172c5dd26 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java @@ -29,11 +29,13 @@ import org.apache.bigtop.manager.server.enums.ApiExceptionEnum; import org.apache.bigtop.manager.server.enums.HealthyStatusEnum; import org.apache.bigtop.manager.server.enums.HostAuthTypeEnum; +import org.apache.bigtop.manager.server.enums.InstalledStatusEnum; import org.apache.bigtop.manager.server.exception.ApiException; import org.apache.bigtop.manager.server.model.converter.HostConverter; import org.apache.bigtop.manager.server.model.dto.HostDTO; import org.apache.bigtop.manager.server.model.query.PageQuery; import org.apache.bigtop.manager.server.model.vo.HostVO; +import org.apache.bigtop.manager.server.model.vo.InstalledStatusVO; import org.apache.bigtop.manager.server.model.vo.PageVO; import org.apache.bigtop.manager.server.service.HostService; import org.apache.bigtop.manager.server.utils.PageUtils; @@ -50,6 +52,9 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.stream.Collectors; @Slf4j @@ -65,6 +70,10 @@ public class HostServiceImpl implements HostService { @Resource private ComponentDao componentDao; + private final List installedStatus = new CopyOnWriteArrayList<>(); + + private final ExecutorService executorService = Executors.newFixedThreadPool(5); + @Override public PageVO list(HostQuery hostQuery) { PageQuery pageQuery = PageUtils.getPageQuery(); @@ -82,7 +91,7 @@ public PageVO list(HostQuery hostQuery) { public List add(HostDTO hostDTO) { List hostPOList = HostConverter.INSTANCE.fromDTO2POListUsingHostnames(hostDTO); for (HostPO hostPO : hostPOList) { - hostPO.setStatus(HealthyStatusEnum.UNKNOWN.getCode()); + hostPO.setStatus(HealthyStatusEnum.HEALTHY.getCode()); } hostDao.saveAll(hostPOList); @@ -165,92 +174,102 @@ public Boolean checkConnection(HostDTO hostDTO) { } @Override - public Boolean installDependencies(List hostIds, String path) { + public Boolean installDependencies(HostDTO hostDTO) { List repoPOList = repoDao.findAll(); Map archRepoMap = repoPOList.stream() .filter(repoPO -> repoPO.getType() == 2) .collect(Collectors.toMap(RepoPO::getArch, repo -> repo)); - List hostPOList = hostDao.findByIds(hostIds); - for (HostPO hostPO : hostPOList) { - HostDTO hostDTO = HostConverter.INSTANCE.fromPO2DTO(hostPO); - - // Get host arch - String arch = execCommandOnRemoteHost(hostDTO, hostDTO.getHostname(), "arch") - .getOutput() - .trim(); - arch = arch.equals("arm64") ? "aarch64" : arch; - - // Download & Extract agent tarball - String repoUrl = archRepoMap.get(arch).getBaseUrl(); - String tarballUrl = repoUrl + "/bigtop-manager-agent.tar.gz"; - String command = "sudo mkdir -p " + path + " &&" - + " sudo chown -R " + hostDTO.getSshUser() + ":" + hostDTO.getSshUser() + " " + path - + " && curl -L " + tarballUrl + " | tar -xz -C " + path; - ShellResult result = execCommandOnRemoteHost(hostDTO, hostDTO.getHostname(), command); - if (result.getExitCode() != MessageConstants.SUCCESS_CODE) { - hostPO.setErrInfo(result.getErrMsg()); - hostDao.updateById(hostPO); - - log.error( - "Unable to download & extract agent tarball, hostname: {}, msg: {}", - hostDTO.getHostname(), - result.getErrMsg()); - throw new ApiException(ApiExceptionEnum.HOST_UNABLE_TO_EXEC_COMMAND, hostDTO.getHostname()); - } + // Clear cache list + installedStatus.clear(); - // Update agent conf - // Current only grpc port needs to be updated if it's not default port - if (hostDTO.getGrpcPort() != 8835) { - command = "sed -i 's/port: 8835/port: " + hostDTO.getGrpcPort() + "/' " + path - + "/bigtop-manager-agent/conf/application.yml"; - result = execCommandOnRemoteHost(hostDTO, hostDTO.getHostname(), command); - if (result.getExitCode() != MessageConstants.SUCCESS_CODE) { - hostPO.setErrInfo(result.getErrMsg()); - hostDao.updateById(hostPO); - - log.error( - "Unable to update agent config, hostname: {}, msg: {}", - hostDTO.getHostname(), - result.getErrMsg()); - throw new ApiException(ApiExceptionEnum.HOST_UNABLE_TO_EXEC_COMMAND, hostDTO.getHostname()); - } - } + for (String hostname : hostDTO.getHostnames()) { + InstalledStatusVO installedStatusVO = new InstalledStatusVO(); + installedStatusVO.setHostname(hostname); + installedStatusVO.setStatus(InstalledStatusEnum.INSTALLING); + installedStatus.add(installedStatusVO); + + // Async install dependencies + executorService.submit(() -> installDependencies(archRepoMap, hostDTO, hostname, installedStatusVO)); + } + + return true; + } - // Run agent in background - command = "nohup " + path + "/bigtop-manager-agent/bin/start.sh --debug > /dev/null 2>&1 &"; - result = execCommandOnRemoteHost(hostDTO, hostDTO.getHostname(), command); + @Override + public List installedStatus() { + return installedStatus; + } + + public void installDependencies( + Map archRepoMap, HostDTO hostDTO, String hostname, InstalledStatusVO installedStatusVO) { + String path = hostDTO.getAgentDir(); + // Get host arch + String arch = + execCommandOnRemoteHost(hostDTO, hostname, "arch").getOutput().trim(); + arch = arch.equals("arm64") ? "aarch64" : arch; + + // Download & Extract agent tarball + String repoUrl = archRepoMap.get(arch).getBaseUrl(); + String tarballUrl = repoUrl + "/bigtop-manager-agent.tar.gz"; + String command = "sudo mkdir -p " + path + " &&" + + " sudo chown -R " + hostDTO.getSshUser() + ":" + hostDTO.getSshUser() + " " + path + + " && curl -L " + tarballUrl + " | tar -xz -C " + path; + ShellResult result = execCommandOnRemoteHost(hostDTO, hostname, command); + if (result.getExitCode() != MessageConstants.SUCCESS_CODE) { + log.error( + "Unable to download & extract agent tarball, hostname: {}, msg: {}", hostname, result.getErrMsg()); + + installedStatusVO.setStatus(InstalledStatusEnum.FAILED); + installedStatusVO.setMessage(result.getErrMsg()); + return; + } + + // Update agent conf + // Current only grpc port needs to be updated if it's not default port + if (hostDTO.getGrpcPort() != 8835) { + command = "sed -i 's/port: 8835/port: " + hostDTO.getGrpcPort() + "/' " + path + + "/bigtop-manager-agent/conf/application.yml"; + result = execCommandOnRemoteHost(hostDTO, hostname, command); if (result.getExitCode() != MessageConstants.SUCCESS_CODE) { - hostPO.setErrInfo(result.getErrMsg()); - hostDao.updateById(hostPO); + log.error("Unable to update agent config, hostname: {}, msg: {}", hostname, result.getErrMsg()); - log.error("Unable to start agent, hostname: {}, msg: {}", hostDTO.getHostname(), result.getErrMsg()); - throw new ApiException(ApiExceptionEnum.HOST_UNABLE_TO_EXEC_COMMAND, hostDTO.getHostname()); + installedStatusVO.setStatus(InstalledStatusEnum.FAILED); + installedStatusVO.setMessage(result.getErrMsg()); + return; } + } - // Check the process, the agent may encounter some errors and exit when starting - // So we need to wait for a while before the check - try { - Thread.sleep(10 * 1000); - } catch (InterruptedException e) { - log.error("Thread sleep interrupted", e); - } - command = "ps -ef | grep bigtop-manager-agent | grep -v grep"; - result = execCommandOnRemoteHost(hostDTO, hostDTO.getHostname(), command); - if (result.getExitCode() != MessageConstants.SUCCESS_CODE - || !result.getOutput().contains("bigtop-manager-agent")) { - hostPO.setErrInfo("Unable to start agent process, please check the log"); - hostDao.updateById(hostPO); - - log.error("Unable to start agent process, hostname: {}", hostDTO.getHostname()); - throw new ApiException(ApiExceptionEnum.HOST_UNABLE_TO_EXEC_COMMAND, hostDTO.getHostname()); - } + // Run agent in background + command = "nohup " + path + "/bigtop-manager-agent/bin/start.sh --debug > /dev/null 2>&1 &"; + result = execCommandOnRemoteHost(hostDTO, hostname, command); + if (result.getExitCode() != MessageConstants.SUCCESS_CODE) { + log.error("Unable to start agent, hostname: {}, msg: {}", hostname, result.getErrMsg()); - hostPO.setStatus(HealthyStatusEnum.HEALTHY.getCode()); - hostDao.updateById(hostPO); + installedStatusVO.setStatus(InstalledStatusEnum.FAILED); + installedStatusVO.setMessage(result.getErrMsg()); + return; } - return true; + // Check the process, the agent may encounter some errors and exit when starting + // So we need to wait for a while before the check + try { + Thread.sleep(10 * 1000); + } catch (InterruptedException e) { + log.error("Thread sleep interrupted", e); + } + command = "ps -ef | grep bigtop-manager-agent | grep -v grep"; + result = execCommandOnRemoteHost(hostDTO, hostname, command); + if (result.getExitCode() != MessageConstants.SUCCESS_CODE + || !result.getOutput().contains("bigtop-manager-agent")) { + log.error("Unable to start agent process, hostname: {}", hostname); + + installedStatusVO.setStatus(InstalledStatusEnum.FAILED); + installedStatusVO.setMessage("Unable to start agent, please check the log."); + return; + } + + installedStatusVO.setStatus(InstalledStatusEnum.SUCCESS); } private ShellResult execCommandOnRemoteHost(HostDTO hostDTO, String hostname, String command) { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/JobServiceImpl.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/JobServiceImpl.java index d316df5ac..f66712caa 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/JobServiceImpl.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/JobServiceImpl.java @@ -160,8 +160,8 @@ private Job recreateJob(JobPO jobPO) { for (int j = 0; j < stage.getTasks().size(); j++) { Task task = stage.getTasks().get(j); - TaskPO taskPO = findCorrectTaskPO( - taskPOList, task.getTaskContext().getHostDTO().getHostname()); + TaskPO taskPO = + findCorrectTaskPO(taskPOList, task.getTaskContext().getHostname()); if (taskPO == null) { throw new ApiException(ApiExceptionEnum.JOB_NOT_RETRYABLE); } diff --git a/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql index 4c1ff01aa..88c94961d 100644 --- a/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql +++ b/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql @@ -91,6 +91,7 @@ CREATE TABLE `host` `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, `cluster_id` BIGINT(20) UNSIGNED DEFAULT NULL, `hostname` VARCHAR(255) DEFAULT NULL, + `agent_dir` VARCHAR(255) DEFAULT NULL, `ssh_user` VARCHAR(255) DEFAULT NULL, `ssh_port` INTEGER DEFAULT NULL, `auth_type` INTEGER DEFAULT NULL COMMENT '1-password, 2-key, 3-no_auth', @@ -143,7 +144,7 @@ CREATE TABLE `service` `user` VARCHAR(255) DEFAULT NULL, `version` VARCHAR(255) DEFAULT NULL, `stack` VARCHAR(255) DEFAULT NULL, - `need_restart` BOOLEAN DEFAULT FALSE, + `restart_flag` BOOLEAN DEFAULT FALSE, `cluster_id` BIGINT, `status` INTEGER DEFAULT NULL COMMENT '1-healthy, 2-unhealthy, 3-unknown', `create_time` DATETIME DEFAULT CURRENT_TIMESTAMP, diff --git a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql index 3d133f72c..a81801445 100644 --- a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql +++ b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql @@ -80,6 +80,7 @@ CREATE TABLE host id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, cluster_id BIGINT DEFAULT NULL, hostname VARCHAR(255) DEFAULT NULL, + agent_dir VARCHAR(255) DEFAULT NULL, ssh_user VARCHAR(255) DEFAULT NULL, ssh_port INT DEFAULT NULL, auth_type INT DEFAULT NULL, @@ -139,7 +140,7 @@ CREATE TABLE service "user" VARCHAR(255) DEFAULT NULL, version VARCHAR(255) DEFAULT NULL, stack VARCHAR(255) DEFAULT NULL, - need_restart BOOLEAN DEFAULT FALSE, + restart_flag BOOLEAN DEFAULT FALSE, cluster_id BIGINT, status INTEGER DEFAULT NULL, create_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml index 0ced8143c..e81dc5596 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml @@ -107,12 +107,22 @@ x86_64 + + + + hadoop-3.3.6-1-x86_64.tar.gz + SHA-256:4727109fe75e87c9611e7ed3c473510fe6cb6c56d28edeaca1de1aeec305bf2e + + + + + aarch64 - hadoop-3.3.6-1.tar.gz - SHA-256:0da6364126c6ea6dd0f5b545afec33957ee801d6db6c0874edb0aa5be1e89995 + hadoop-3.3.6-1-aarch64.tar.gz + SHA-256:2a9312eac69a542c6266f1689d29ded243dde2d61add333a9762829008f97442