From d7c3db227b2c6ade297cbbd3a6a4c3d6b51aadb2 Mon Sep 17 00:00:00 2001 From: lhpqaq <657407891@qq.com> Date: Wed, 30 Oct 2024 19:56:28 +0800 Subject: [PATCH 1/6] support openeuler22 arm --- .../resources/stacks/infra/1.0.0/metainfo.xml | 8 ++++++++ .../1.0.0/services/prometheus/metainfo.xml | 14 ++++++++++++++ .../manager/stack/core/param/BaseParams.java | 2 +- .../stack/core/spi/script/AbstractScript.java | 1 - .../prometheus/PrometheusServerScript.java | 19 ++++++++++++++++--- 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/metainfo.xml index 74a38f3dd..68a9a0847 100644 --- a/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/metainfo.xml @@ -36,6 +36,14 @@ x86_64 http://123.56.2.244/infra/1.0.0/tarballs/ + + Infra-1.0.0 + Infra-1.0.0 for RockyLinux-8 x86_64 + tarball + openeuler22 + aarch64 + http://127.0.0.1/infra/1.0.0/tarballs + \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/prometheus/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/prometheus/metainfo.xml index 97cc63e6f..7c067cca9 100644 --- a/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/prometheus/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/prometheus/metainfo.xml @@ -61,6 +61,20 @@ + + + openeuler22 + + + aarch64 + + + + prometheus-2.55.0-rc.0.linux-arm64.tar.gz + SHA-256:a25d8fae86ff1acd95e70405d1227142d1a8a214767c2db26a2289c97b80ec2b + + + \ No newline at end of file diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/param/BaseParams.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/param/BaseParams.java index d5df7a466..e17f47de3 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/param/BaseParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/param/BaseParams.java @@ -106,7 +106,7 @@ public RepoInfo repo() { .filter(r -> OSDetection.getArch().equals(r.getArch())) .findFirst() .orElseThrow(() -> new StackException( - "Cannot find repo for os: [{}] and arch: [{}]", OSDetection.getOS(), OSDetection.getArch())); + "Cannot find repo for os: [{0}] and arch: [{1}]", OSDetection.getOS(), OSDetection.getArch())); } @Override diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java index 94733d4b9..9797f19b5 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java @@ -46,7 +46,6 @@ public ShellResult install(Params params, Properties properties) { List packages = params.packages(); String stackHome = params.stackHome(); String serviceHome = params.serviceHome(); - for (PackageInfo packageInfo : packages) { Integer skipLevels = Integer.parseInt(properties.getProperty(PROPERTY_KEY_SKIP_LEVELS, "0")); TarballUtils.installPackage(repo.getBaseUrl(), stackHome, serviceHome, packageInfo, skipLevels); diff --git a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java index 581ddb67f..b90c872b2 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java @@ -63,9 +63,21 @@ public ShellResult start(Params params) { @Override public ShellResult stop(Params params) { PrometheusParams prometheusParams = (PrometheusParams) params; - String cmd = "pkill -f prometheus"; + String cmd = MessageFormat.format("pgrep -f {0}/prometheus", prometheusParams.serviceHome()); try { - return LinuxOSUtils.sudoExecCmd(cmd, prometheusParams.user()); + LinuxOSUtils.sudoExecCmd(cmd, prometheusParams.user()); + long startTime = System.currentTimeMillis(); + long maxWaitTime = 5000; + long pollInterval = 500; + + while (System.currentTimeMillis() - startTime < maxWaitTime) { + ShellResult statusResult = status(params); + if (statusResult.getExitCode() == 0) { + return statusResult; + } + Thread.sleep(pollInterval); + } + return status(params); } catch (Exception e) { throw new StackException(e); } @@ -73,7 +85,8 @@ public ShellResult stop(Params params) { @Override public ShellResult status(Params params) { - String cmd = "pgrep -f prometheus"; + PrometheusParams prometheusParams = (PrometheusParams) params; + String cmd = MessageFormat.format("pgrep -f {0}/prometheus", prometheusParams.serviceHome()); try { ShellResult result = LinuxOSUtils.execCmd(cmd); if (result.getExitCode() == 0) { From fe15c22f6fd662b6ca72641acac7ac1af9e2f49e Mon Sep 17 00:00:00 2001 From: lhpqaq <657407891@qq.com> Date: Wed, 30 Oct 2024 20:00:13 +0800 Subject: [PATCH 2/6] fix --- .../prometheus/PrometheusServerScript.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java index b90c872b2..a1f3c871f 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java @@ -53,17 +53,6 @@ public ShellResult start(Params params) { String cmd = MessageFormat.format( "nohup {0}/prometheus --config.file={0}/prometheus.yml --storage.tsdb.path={0}/data > {0}/nohup.out 2>&1 &", prometheusParams.serviceHome()); - try { - return LinuxOSUtils.sudoExecCmd(cmd, prometheusParams.user()); - } catch (Exception e) { - throw new StackException(e); - } - } - - @Override - public ShellResult stop(Params params) { - PrometheusParams prometheusParams = (PrometheusParams) params; - String cmd = MessageFormat.format("pgrep -f {0}/prometheus", prometheusParams.serviceHome()); try { LinuxOSUtils.sudoExecCmd(cmd, prometheusParams.user()); long startTime = System.currentTimeMillis(); @@ -83,6 +72,17 @@ public ShellResult stop(Params params) { } } + @Override + public ShellResult stop(Params params) { + PrometheusParams prometheusParams = (PrometheusParams) params; + String cmd = MessageFormat.format("pgrep -f {0}/prometheus", prometheusParams.serviceHome()); + try { + return LinuxOSUtils.sudoExecCmd(cmd, prometheusParams.user()); + } catch (Exception e) { + throw new StackException(e); + } + } + @Override public ShellResult status(Params params) { PrometheusParams prometheusParams = (PrometheusParams) params; From 50a1f681d7fa3c03823ef030b77d318f342060c9 Mon Sep 17 00:00:00 2001 From: lhpqaq <657407891@qq.com> Date: Wed, 30 Oct 2024 20:02:28 +0800 Subject: [PATCH 3/6] fix --- .../src/main/resources/stacks/infra/1.0.0/metainfo.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/metainfo.xml index 68a9a0847..a80aed4b5 100644 --- a/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/metainfo.xml @@ -38,7 +38,7 @@ Infra-1.0.0 - Infra-1.0.0 for RockyLinux-8 x86_64 + Infra-1.0.0 for openEuler-22.03 aarch64 tarball openeuler22 aarch64 From 123cf3198467b7b8d76904d5e925eb398b187737 Mon Sep 17 00:00:00 2001 From: lhpqaq <657407891@qq.com> Date: Wed, 30 Oct 2024 20:08:50 +0800 Subject: [PATCH 4/6] fix --- .../bigtop/manager/stack/core/spi/script/AbstractScript.java | 1 + .../stack/infra/v1_0_0/prometheus/PrometheusServerScript.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java index 9797f19b5..94733d4b9 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java @@ -46,6 +46,7 @@ public ShellResult install(Params params, Properties properties) { List packages = params.packages(); String stackHome = params.stackHome(); String serviceHome = params.serviceHome(); + for (PackageInfo packageInfo : packages) { Integer skipLevels = Integer.parseInt(properties.getProperty(PROPERTY_KEY_SKIP_LEVELS, "0")); TarballUtils.installPackage(repo.getBaseUrl(), stackHome, serviceHome, packageInfo, skipLevels); diff --git a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java index a1f3c871f..c93416079 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java @@ -75,7 +75,7 @@ public ShellResult start(Params params) { @Override public ShellResult stop(Params params) { PrometheusParams prometheusParams = (PrometheusParams) params; - String cmd = MessageFormat.format("pgrep -f {0}/prometheus", prometheusParams.serviceHome()); + String cmd = MessageFormat.format("pkill -f {0}/prometheus", prometheusParams.serviceHome()); try { return LinuxOSUtils.sudoExecCmd(cmd, prometheusParams.user()); } catch (Exception e) { From c36b1fe15df8d4268d83e0341e8dfde2791e4aec Mon Sep 17 00:00:00 2001 From: lhpqaq <657407891@qq.com> Date: Thu, 31 Oct 2024 10:40:54 +0800 Subject: [PATCH 5/6] Squashed commit of the following: commit 2f1e553c6e7d610d3e7d42e3169a5819e6815a65 Author: Zhiguo Wu Date: Thu Oct 31 10:10:06 2024 +0800 BIGTOP-4253: Refactor APIs for Stack (#96) --- .../bigtop/manager/dao/po/ClusterPO.java | 3 - .../apache/bigtop/manager/dao/po/StackPO.java | 43 --- .../manager/dao/repository/StackDao.java | 30 -- .../resources/mapper/mysql/StackMapper.xml | 46 --- .../mapper/postgresql/StackMapper.xml | 46 --- .../server/command/job/AbstractJob.java | 3 - .../command/job/AbstractServiceJob.java | 26 +- .../server/command/job/ServiceInstallJob.java | 4 +- .../command/task/CacheFileUpdateTask.java | 14 - .../validator/RequiredServicesValidator.java | 9 +- .../command/validator/StackValidator.java | 57 ---- .../server/controller/StackController.java | 17 - .../model/converter/ClusterConverter.java | 9 - .../model/converter/ServiceConverter.java | 12 - .../model/converter/StackConverter.java | 6 - .../manager/server/model/dto/ServiceDTO.java | 2 + .../manager/server/model/dto/StackDTO.java | 18 +- .../manager/server/model/vo/ServiceVO.java | 4 +- .../manager/server/model/vo/StackVO.java | 2 - .../manager/server/service/StackService.java | 6 - .../service/impl/ClusterServiceImpl.java | 14 +- .../server/service/impl/StackServiceImpl.java | 57 +--- .../server/stack/StackInitializer.java | 73 ----- .../server/stack/model/StackModel.java | 51 --- .../server/stack/xml/StackMetainfoXml.java | 37 --- .../server/utils/StackConfigUtils.java | 6 +- .../manager/server/utils/StackUtils.java | 306 +++++++----------- .../main/resources/ddl/MySQL-DDL-CREATE.sql | 33 +- .../resources/ddl/PostgreSQL-DDL-CREATE.sql | 15 - .../stacks/bigtop/3.3.0/metainfo.xml | 61 ---- .../resources/stacks/infra/1.0.0/metainfo.xml | 49 --- .../controller/StackControllerTest.java | 28 -- .../server/service/ClusterServiceTest.java | 32 +- 33 files changed, 160 insertions(+), 959 deletions(-) delete mode 100644 bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/StackPO.java delete mode 100644 bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/StackDao.java delete mode 100644 bigtop-manager-dao/src/main/resources/mapper/mysql/StackMapper.xml delete mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/StackMapper.xml delete mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/validator/StackValidator.java delete mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/StackInitializer.java delete mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/StackModel.java delete mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/StackMetainfoXml.java delete mode 100644 bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml delete mode 100644 bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/metainfo.xml diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ClusterPO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ClusterPO.java index 3b70c0af2..3076e22ca 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ClusterPO.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ClusterPO.java @@ -60,9 +60,6 @@ public class ClusterPO extends BasePO implements Serializable { @Column(name = "selected") private Boolean selected; - @Column(name = "stack_id") - private Long stackId; - @Transient @Column(name = "stack_name") private String stackName; diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/StackPO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/StackPO.java deleted file mode 100644 index 5b7add256..000000000 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/StackPO.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bigtop.manager.dao.po; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -import jakarta.persistence.Column; -import jakarta.persistence.Id; -import jakarta.persistence.Table; -import java.io.Serializable; - -@Data -@EqualsAndHashCode(callSuper = true) -@Table(name = "stack") -public class StackPO extends BasePO implements Serializable { - - @Id - @Column(name = "id") - private Long id; - - @Column(name = "stack_name", nullable = false) - private String stackName; - - @Column(name = "stack_version", nullable = false) - private String stackVersion; -} diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/StackDao.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/StackDao.java deleted file mode 100644 index 8523d2be1..000000000 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/StackDao.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.bigtop.manager.dao.repository; - -import org.apache.bigtop.manager.dao.po.StackPO; - -import org.apache.ibatis.annotations.Param; - -public interface StackDao extends BaseDao { - - StackPO findByStackNameAndStackVersion( - @Param("stackName") String stackName, @Param("stackVersion") String stackVersion); -} diff --git a/bigtop-manager-dao/src/main/resources/mapper/mysql/StackMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/mysql/StackMapper.xml deleted file mode 100644 index d654df909..000000000 --- a/bigtop-manager-dao/src/main/resources/mapper/mysql/StackMapper.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - - - id, stack_name, stack_version - - - - - \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/StackMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/StackMapper.xml deleted file mode 100644 index d9da21ba5..000000000 --- a/bigtop-manager-dao/src/main/resources/mapper/postgresql/StackMapper.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - - - "id", "stack_name", "stack_version" - - - - - \ No newline at end of file diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java index 6ae1ee859..bf5d3475e 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java @@ -26,7 +26,6 @@ import org.apache.bigtop.manager.dao.po.TaskPO; import org.apache.bigtop.manager.dao.repository.ClusterDao; import org.apache.bigtop.manager.dao.repository.JobDao; -import org.apache.bigtop.manager.dao.repository.StackDao; import org.apache.bigtop.manager.dao.repository.StageDao; import org.apache.bigtop.manager.dao.repository.TaskDao; import org.apache.bigtop.manager.server.command.stage.Stage; @@ -42,7 +41,6 @@ @Slf4j public abstract class AbstractJob implements Job { - protected StackDao stackDao; protected ClusterDao clusterDao; protected JobDao jobDao; protected StageDao stageDao; @@ -70,7 +68,6 @@ public AbstractJob(JobContext jobContext) { } protected void injectBeans() { - this.stackDao = SpringContextHolder.getBean(StackDao.class); this.clusterDao = SpringContextHolder.getBean(ClusterDao.class); this.jobDao = SpringContextHolder.getBean(JobDao.class); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java index 5192d6118..9014fadae 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java @@ -24,7 +24,6 @@ import org.apache.bigtop.manager.dao.po.ComponentPO; import org.apache.bigtop.manager.dao.po.HostComponentPO; import org.apache.bigtop.manager.dao.po.HostPO; -import org.apache.bigtop.manager.dao.po.StackPO; import org.apache.bigtop.manager.dao.repository.ComponentDao; import org.apache.bigtop.manager.dao.repository.HostComponentDao; import org.apache.bigtop.manager.dao.repository.HostDao; @@ -40,6 +39,7 @@ import org.apache.bigtop.manager.server.model.dto.ComponentDTO; import org.apache.bigtop.manager.server.model.dto.ComponentHostDTO; import org.apache.bigtop.manager.server.model.dto.ServiceDTO; +import org.apache.bigtop.manager.server.model.dto.StackDTO; import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO; import org.apache.bigtop.manager.server.stack.dag.ComponentCommandWrapper; import org.apache.bigtop.manager.server.stack.dag.DAG; @@ -57,9 +57,10 @@ public abstract class AbstractServiceJob extends AbstractJob { protected HostComponentDao hostComponentDao; protected HostDao hostDao; - protected String stackName; - protected String stackVersion; - protected DAG dag; + // TODO: temp code + protected String stackName = "bigtop"; + protected String stackVersion = "3.3.0"; + protected DAG dag = StackUtils.DAG; public AbstractServiceJob(JobContext jobContext) { super(jobContext); @@ -77,18 +78,13 @@ protected void injectBeans() { @Override protected void beforeCreateStages() { super.beforeCreateStages(); - StackPO stackPO = stackDao.findById(clusterPO.getStackId()); - - stackName = stackPO.getStackName(); - stackVersion = stackPO.getStackVersion(); - dag = StackUtils.getStackDagMap().get(StackUtils.fullStackName(stackName, stackVersion)); } protected StageContext createStageContext(String serviceName, String componentName, List hostnames) { StageContext stageContext = StageContext.fromCommandDTO(jobContext.getCommandDTO()); - ServiceDTO serviceDTO = StackUtils.getServiceDTO(stackName, stackVersion, serviceName); - ComponentDTO componentDTO = StackUtils.getComponentDTO(stackName, stackVersion, componentName); + ServiceDTO serviceDTO = StackUtils.getServiceDTO(serviceName); + ComponentDTO componentDTO = StackUtils.getComponentDTO(componentName); stageContext.setServiceDTO(serviceDTO); stageContext.setComponentDTO(componentDTO); @@ -126,7 +122,7 @@ protected List getComponentNames() { } protected String findServiceNameByComponentName(String componentName) { - for (ServiceDTO serviceDTO : StackUtils.getServiceDTOList(stackName, stackVersion)) { + for (ServiceDTO serviceDTO : StackUtils.getServiceDTOList(new StackDTO(stackName, stackVersion))) { for (ComponentDTO componentDTO : serviceDTO.getComponents()) { if (componentDTO.getComponentName().equals(componentName)) { return serviceDTO.getServiceName(); @@ -138,17 +134,17 @@ protected String findServiceNameByComponentName(String componentName) { } protected Boolean isMasterComponent(String componentName) { - ComponentDTO componentDTO = StackUtils.getComponentDTO(stackName, stackVersion, componentName); + ComponentDTO componentDTO = StackUtils.getComponentDTO(componentName); return componentDTO.getCategory().equalsIgnoreCase(ComponentCategories.MASTER); } protected Boolean isSlaveComponent(String componentName) { - ComponentDTO componentDTO = StackUtils.getComponentDTO(stackName, stackVersion, componentName); + ComponentDTO componentDTO = StackUtils.getComponentDTO(componentName); return componentDTO.getCategory().equalsIgnoreCase(ComponentCategories.SLAVE); } protected Boolean isClientComponent(String componentName) { - ComponentDTO componentDTO = StackUtils.getComponentDTO(stackName, stackVersion, componentName); + ComponentDTO componentDTO = StackUtils.getComponentDTO(componentName); return componentDTO.getCategory().equalsIgnoreCase(ComponentCategories.CLIENT); } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceInstallJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceInstallJob.java index 100ccdf41..c701f8c76 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceInstallJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceInstallJob.java @@ -144,7 +144,7 @@ private void upsertService(ServicePO servicePO, ServiceCommandDTO serviceCommand // 1. Persist service and components if (servicePO == null) { - ServiceDTO serviceDTO = StackUtils.getServiceDTO(stackName, stackVersion, serviceName); + ServiceDTO serviceDTO = StackUtils.getServiceDTO(serviceName); servicePO = ServiceConverter.INSTANCE.fromDTO2PO(serviceDTO, clusterPO); serviceDao.save(servicePO); } @@ -158,7 +158,7 @@ private void upsertService(ServicePO servicePO, ServiceCommandDTO serviceCommand // 3. Persist component ComponentPO componentPO = componentDao.findByClusterIdAndComponentName(clusterId, componentName); if (componentPO == null) { - ComponentDTO componentDTO = StackUtils.getComponentDTO(stackName, stackVersion, componentName); + ComponentDTO componentDTO = StackUtils.getComponentDTO(componentName); componentPO = ComponentConverter.INSTANCE.fromDTO2PO(componentDTO, servicePO, clusterPO); componentDao.save(componentPO); } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/CacheFileUpdateTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/CacheFileUpdateTask.java index 4d6790655..70b969a4d 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/CacheFileUpdateTask.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/CacheFileUpdateTask.java @@ -48,12 +48,7 @@ import org.apache.bigtop.manager.server.model.converter.RepoConverter; import org.apache.bigtop.manager.server.model.dto.PropertyDTO; import org.apache.bigtop.manager.server.model.dto.RepoDTO; -import org.apache.bigtop.manager.server.model.dto.ServiceDTO; -import org.apache.bigtop.manager.server.model.dto.StackDTO; import org.apache.bigtop.manager.server.utils.StackConfigUtils; -import org.apache.bigtop.manager.server.utils.StackUtils; - -import org.apache.commons.lang3.tuple.ImmutablePair; import com.fasterxml.jackson.core.type.TypeReference; @@ -213,11 +208,6 @@ private void genEmptyCaches() { userMap = new HashMap<>(); settingsMap = new HashMap<>(); - String fullStackName = StackUtils.fullStackName(taskContext.getStackName(), taskContext.getStackVersion()); - ImmutablePair> immutablePair = - StackUtils.getStackKeyMap().get(fullStackName); - StackDTO stackDTO = immutablePair.getLeft(); - Map properties = taskContext.getProperties(); repoList = RepoConverter.INSTANCE.fromDTO2Message((List) properties.get("repoInfoList")); @@ -225,10 +215,6 @@ private void genEmptyCaches() { clusterInfo.setClusterName(taskContext.getClusterName()); clusterInfo.setStackName(taskContext.getStackName()); clusterInfo.setStackVersion(taskContext.getStackVersion()); - clusterInfo.setUserGroup(stackDTO.getUserGroup()); - clusterInfo.setRepoTemplate(stackDTO.getRepoTemplate()); - clusterInfo.setRoot(stackDTO.getRoot()); - clusterInfo.setPackages(List.of(stackDTO.getPackages().split(","))); List hostnames = (List) properties.get("hostnames"); hostMap.put(Constants.ALL_HOST_KEY, new HashSet<>(hostnames)); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/validator/RequiredServicesValidator.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/validator/RequiredServicesValidator.java index 0daefc717..c2073fd04 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/validator/RequiredServicesValidator.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/validator/RequiredServicesValidator.java @@ -19,7 +19,6 @@ package org.apache.bigtop.manager.server.command.validator; import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.dao.po.ClusterPO; import org.apache.bigtop.manager.dao.po.ServicePO; import org.apache.bigtop.manager.dao.repository.ClusterDao; import org.apache.bigtop.manager.dao.repository.ServiceDao; @@ -37,6 +36,7 @@ import org.springframework.stereotype.Component; import jakarta.annotation.Resource; +import java.util.HashSet; import java.util.List; @Component @@ -59,15 +59,12 @@ public void validate(ValidatorContext context) { List serviceCommands = commandDTO.getServiceCommands(); Long clusterId = commandDTO.getClusterId(); - ClusterPO clusterPO = clusterDao.findByIdJoin(clusterId); - String stackName = clusterPO.getStackName(); - String stackVersion = clusterPO.getStackVersion(); List serviceNames = serviceCommands.stream().map(ServiceCommandDTO::getServiceName).toList(); for (ServiceCommandDTO serviceCommand : serviceCommands) { String serviceName = serviceCommand.getServiceName(); - ServiceDTO serviceDTO = StackUtils.getServiceDTO(stackName, stackVersion, serviceName); + ServiceDTO serviceDTO = StackUtils.getServiceDTO(serviceName); List requiredServices = serviceDTO.getRequiredServices(); if (CollectionUtils.isEmpty(requiredServices)) { return; @@ -79,7 +76,7 @@ public void validate(ValidatorContext context) { requiredServices.removeAll(list); - if (!serviceNames.containsAll(requiredServices)) { + if (!new HashSet<>(serviceNames).containsAll(requiredServices)) { throw new ApiException(ApiExceptionEnum.SERVICE_REQUIRED_NOT_FOUND, String.join(",", requiredServices)); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/validator/StackValidator.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/validator/StackValidator.java deleted file mode 100644 index 387462d58..000000000 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/validator/StackValidator.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bigtop.manager.server.command.validator; - -import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.dao.po.StackPO; -import org.apache.bigtop.manager.dao.repository.StackDao; -import org.apache.bigtop.manager.server.command.CommandIdentifier; -import org.apache.bigtop.manager.server.enums.ApiExceptionEnum; -import org.apache.bigtop.manager.server.enums.CommandLevel; -import org.apache.bigtop.manager.server.exception.ApiException; -import org.apache.bigtop.manager.server.model.dto.command.ClusterCommandDTO; - -import org.springframework.stereotype.Component; - -import jakarta.annotation.Resource; -import java.util.List; - -@Component -public class StackValidator implements CommandValidator { - - @Resource - private StackDao stackDao; - - @Override - public List getCommandIdentifiers() { - return List.of(new CommandIdentifier(CommandLevel.CLUSTER, Command.CREATE)); - } - - @Override - public void validate(ValidatorContext context) { - ClusterCommandDTO clusterCommand = context.getCommandDTO().getClusterCommand(); - String stackName = clusterCommand.getStackName(); - String stackVersion = clusterCommand.getStackVersion(); - - StackPO stackPO = stackDao.findByStackNameAndStackVersion(stackName, stackVersion); - if (stackPO == null) { - throw new ApiException(ApiExceptionEnum.STACK_NOT_FOUND); - } - } -} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/controller/StackController.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/controller/StackController.java index ef30f3201..b6a9e9678 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/controller/StackController.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/controller/StackController.java @@ -18,14 +18,11 @@ */ package org.apache.bigtop.manager.server.controller; -import org.apache.bigtop.manager.server.model.vo.ServiceComponentVO; -import org.apache.bigtop.manager.server.model.vo.ServiceConfigVO; import org.apache.bigtop.manager.server.model.vo.StackVO; import org.apache.bigtop.manager.server.service.StackService; import org.apache.bigtop.manager.server.utils.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -48,18 +45,4 @@ public class StackController { public ResponseEntity> list() { return ResponseEntity.success(stackService.list()); } - - @Operation(summary = "list", description = "List stacks components") - @GetMapping("/{stackName}/{stackVersion}/components") - public ResponseEntity> components( - @PathVariable String stackName, @PathVariable String stackVersion) { - return ResponseEntity.success(stackService.components(stackName, stackVersion)); - } - - @Operation(summary = "list", description = "List stacks configurations") - @GetMapping("/{stackName}/{stackVersion}/configurations") - public ResponseEntity> configurations( - @PathVariable String stackName, @PathVariable String stackVersion) { - return ResponseEntity.success(stackService.configurations(stackName, stackVersion)); - } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ClusterConverter.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ClusterConverter.java index 70f909341..d9e86f627 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ClusterConverter.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ClusterConverter.java @@ -19,17 +19,13 @@ package org.apache.bigtop.manager.server.model.converter; import org.apache.bigtop.manager.dao.po.ClusterPO; -import org.apache.bigtop.manager.dao.po.StackPO; import org.apache.bigtop.manager.server.config.MapStructSharedConfig; import org.apache.bigtop.manager.server.model.dto.ClusterDTO; -import org.apache.bigtop.manager.server.model.dto.StackDTO; import org.apache.bigtop.manager.server.model.dto.command.ClusterCommandDTO; import org.apache.bigtop.manager.server.model.req.ClusterReq; import org.apache.bigtop.manager.server.model.vo.ClusterVO; -import org.mapstruct.Context; import org.mapstruct.Mapper; -import org.mapstruct.Mapping; import org.mapstruct.factory.Mappers; @Mapper(config = MapStructSharedConfig.class) @@ -46,9 +42,4 @@ public interface ClusterConverter { ClusterDTO fromPO2DTO(ClusterPO clusterPO); ClusterVO fromEntity2VO(ClusterPO clusterPO); - - @Mapping(target = "stackName", expression = "java(stackPO.getStackName())") - @Mapping(target = "stackVersion", expression = "java(stackPO.getStackVersion())") - @Mapping(target = "stackId", expression = "java(stackPO.getId())") - ClusterPO fromDTO2PO(ClusterDTO clusterDTO, StackDTO stackDTO, @Context StackPO stackPO); } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ServiceConverter.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ServiceConverter.java index b12b85223..2abd3c52a 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ServiceConverter.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ServiceConverter.java @@ -22,7 +22,6 @@ import org.apache.bigtop.manager.dao.po.ServicePO; import org.apache.bigtop.manager.server.config.MapStructSharedConfig; import org.apache.bigtop.manager.server.model.dto.ServiceDTO; -import org.apache.bigtop.manager.server.model.dto.StackDTO; import org.apache.bigtop.manager.server.model.vo.ServiceVO; import org.apache.bigtop.manager.server.stack.model.ServiceModel; @@ -32,7 +31,6 @@ import org.mapstruct.factory.Mappers; import java.util.List; -import java.util.stream.Collectors; @Mapper( uses = {ComponentConverter.class, ConverterTool.class}, @@ -51,15 +49,6 @@ public interface ServiceConverter { List fromDTO2VO(List serviceDTOList); - @Mapping(target = "serviceGroup", source = "stackDTO.userGroup") - ServiceVO fromDTO2VO(ServiceDTO serviceDTO, StackDTO stackDTO); - - default List fromDTO2VO(List serviceDTOList, StackDTO stackDTO) { - return serviceDTOList.stream() - .map(serviceDTO -> fromDTO2VO(serviceDTO, stackDTO)) - .collect(Collectors.toList()); - } - @Mapping(target = "serviceName", source = "name") @Mapping(target = "serviceDesc", source = "desc") @Mapping(target = "serviceVersion", source = "version") @@ -67,7 +56,6 @@ default List fromDTO2VO(List serviceDTOList, StackDTO sta ServiceDTO fromModel2DTO(ServiceModel serviceModel); @Mapping(target = "requiredServices", source = "requiredServices", qualifiedByName = "json2List") - @Mapping(target = "serviceGroup", source = "userGroup") ServiceVO fromPO2VO(ServicePO servicePO); List fromPO2VO(List servicePOList); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/StackConverter.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/StackConverter.java index 2db76ccd5..e954e3199 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/StackConverter.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/StackConverter.java @@ -18,11 +18,9 @@ */ package org.apache.bigtop.manager.server.model.converter; -import org.apache.bigtop.manager.dao.po.StackPO; import org.apache.bigtop.manager.server.config.MapStructSharedConfig; import org.apache.bigtop.manager.server.model.dto.StackDTO; import org.apache.bigtop.manager.server.model.vo.StackVO; -import org.apache.bigtop.manager.server.stack.model.StackModel; import org.mapstruct.Mapper; import org.mapstruct.factory.Mappers; @@ -32,9 +30,5 @@ public interface StackConverter { StackConverter INSTANCE = Mappers.getMapper(StackConverter.class); - StackVO fromPO2VO(StackPO stackPO); - StackVO fromDTO2VO(StackDTO stackDTO); - - StackDTO fromModel2DTO(StackModel stackModel); } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/ServiceDTO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/ServiceDTO.java index cb742eee2..f4440b4c2 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/ServiceDTO.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/ServiceDTO.java @@ -39,5 +39,7 @@ public class ServiceDTO { private List components; + private List configs; + private List requiredServices; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/StackDTO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/StackDTO.java index 9ad94ac39..3c5fb0e5f 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/StackDTO.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/StackDTO.java @@ -18,26 +18,16 @@ */ package org.apache.bigtop.manager.server.model.dto; +import lombok.AllArgsConstructor; import lombok.Data; - -import java.util.List; +import lombok.EqualsAndHashCode; @Data +@AllArgsConstructor +@EqualsAndHashCode public class StackDTO { private String stackName; private String stackVersion; - - private String scope; - - private String root; - - private String userGroup; - - private String packages; - - private String repoTemplate; - - private List repos; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/ServiceVO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/ServiceVO.java index 431095119..e82787bfa 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/ServiceVO.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/ServiceVO.java @@ -39,7 +39,9 @@ public class ServiceVO { private String serviceUser; - private String serviceGroup; + private List components; + + private List configs; private List requiredServices; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/StackVO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/StackVO.java index ac20407d5..7073bf4f6 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/StackVO.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/StackVO.java @@ -30,6 +30,4 @@ public class StackVO { private String stackVersion; private List services; - - private List repos; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/StackService.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/StackService.java index 3aac6c987..d010852ee 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/StackService.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/StackService.java @@ -18,8 +18,6 @@ */ package org.apache.bigtop.manager.server.service; -import org.apache.bigtop.manager.server.model.vo.ServiceComponentVO; -import org.apache.bigtop.manager.server.model.vo.ServiceConfigVO; import org.apache.bigtop.manager.server.model.vo.StackVO; import java.util.List; @@ -32,8 +30,4 @@ public interface StackService { * @return Stacks */ List list(); - - List components(String stackName, String stackVersion); - - List configurations(String stackName, String stackVersion); } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ClusterServiceImpl.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ClusterServiceImpl.java index 3bc0403c7..6c1982ea4 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ClusterServiceImpl.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ClusterServiceImpl.java @@ -21,20 +21,16 @@ import org.apache.bigtop.manager.common.enums.MaintainState; import org.apache.bigtop.manager.dao.po.ClusterPO; import org.apache.bigtop.manager.dao.po.RepoPO; -import org.apache.bigtop.manager.dao.po.StackPO; import org.apache.bigtop.manager.dao.repository.ClusterDao; import org.apache.bigtop.manager.dao.repository.RepoDao; -import org.apache.bigtop.manager.dao.repository.StackDao; import org.apache.bigtop.manager.server.enums.ApiExceptionEnum; import org.apache.bigtop.manager.server.exception.ApiException; import org.apache.bigtop.manager.server.model.converter.ClusterConverter; import org.apache.bigtop.manager.server.model.converter.RepoConverter; import org.apache.bigtop.manager.server.model.dto.ClusterDTO; -import org.apache.bigtop.manager.server.model.dto.StackDTO; import org.apache.bigtop.manager.server.model.vo.ClusterVO; import org.apache.bigtop.manager.server.service.ClusterService; import org.apache.bigtop.manager.server.service.HostService; -import org.apache.bigtop.manager.server.utils.StackUtils; import org.springframework.stereotype.Service; @@ -54,9 +50,6 @@ public class ClusterServiceImpl implements ClusterService { @Resource private RepoDao repoDao; - @Resource - private StackDao stackDao; - @Resource private HostService hostService; @@ -74,12 +67,7 @@ public List list() { @Override public ClusterVO save(ClusterDTO clusterDTO) { // Save cluster - StackPO stackPO = - stackDao.findByStackNameAndStackVersion(clusterDTO.getStackName(), clusterDTO.getStackVersion()); - StackDTO stackDTO = StackUtils.getStackKeyMap() - .get(StackUtils.fullStackName(clusterDTO.getStackName(), clusterDTO.getStackVersion())) - .getLeft(); - ClusterPO clusterPO = ClusterConverter.INSTANCE.fromDTO2PO(clusterDTO, stackDTO, stackPO); + ClusterPO clusterPO = ClusterConverter.INSTANCE.fromDTO2PO(clusterDTO); clusterPO.setSelected(clusterDao.count() == 0); clusterPO.setState(MaintainState.UNINSTALLED.getName()); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/StackServiceImpl.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/StackServiceImpl.java index 36bbef0c7..e6b421f33 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/StackServiceImpl.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/StackServiceImpl.java @@ -18,23 +18,15 @@ */ package org.apache.bigtop.manager.server.service.impl; -import org.apache.bigtop.manager.server.enums.ApiExceptionEnum; -import org.apache.bigtop.manager.server.exception.ApiException; -import org.apache.bigtop.manager.server.model.converter.ComponentConverter; import org.apache.bigtop.manager.server.model.converter.ServiceConverter; import org.apache.bigtop.manager.server.model.converter.StackConverter; -import org.apache.bigtop.manager.server.model.converter.TypeConfigConverter; import org.apache.bigtop.manager.server.model.dto.ServiceDTO; import org.apache.bigtop.manager.server.model.dto.StackDTO; import org.apache.bigtop.manager.server.model.dto.TypeConfigDTO; -import org.apache.bigtop.manager.server.model.vo.ServiceComponentVO; -import org.apache.bigtop.manager.server.model.vo.ServiceConfigVO; import org.apache.bigtop.manager.server.model.vo.StackVO; import org.apache.bigtop.manager.server.service.StackService; import org.apache.bigtop.manager.server.utils.StackUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; - import org.springframework.stereotype.Service; import lombok.extern.slf4j.Slf4j; @@ -50,11 +42,14 @@ public class StackServiceImpl implements StackService { @Override public List list() { List stackVOList = new ArrayList<>(); - Map>> stackKeyMap = StackUtils.getStackKeyMap(); - for (ImmutablePair> pair : stackKeyMap.values()) { - StackDTO stackDTO = pair.left; - List serviceDTOList = pair.right; + for (Map.Entry> entry : StackUtils.STACK_SERVICE_MAP.entrySet()) { + StackDTO stackDTO = entry.getKey(); + List serviceDTOList = entry.getValue(); + for (ServiceDTO serviceDTO : serviceDTOList) { + List configs = StackUtils.SERVICE_CONFIG_MAP.get(serviceDTO.getServiceName()); + serviceDTO.setConfigs(configs); + } StackVO stackVO = StackConverter.INSTANCE.fromDTO2VO(stackDTO); stackVO.setServices(ServiceConverter.INSTANCE.fromDTO2VO(serviceDTOList)); @@ -63,42 +58,4 @@ public List list() { return stackVOList; } - - @Override - public List components(String stackName, String stackVersion) { - List list = new ArrayList<>(); - - ImmutablePair> pair = - StackUtils.getStackKeyMap().get(StackUtils.fullStackName(stackName, stackVersion)); - if (pair == null) { - throw new ApiException(ApiExceptionEnum.STACK_NOT_FOUND); - } - - List serviceDTOList = pair.right; - for (ServiceDTO serviceDTO : serviceDTOList) { - ServiceComponentVO element = new ServiceComponentVO(); - element.setServiceName(serviceDTO.getServiceName()); - element.setComponents(ComponentConverter.INSTANCE.fromDTO2VO(serviceDTO.getComponents())); - list.add(element); - } - - return list; - } - - @Override - public List configurations(String stackName, String stackVersion) { - List list = new ArrayList<>(); - Map>> stackConfigMap = StackUtils.getStackConfigMap(); - Map> serviceConfigMap = - stackConfigMap.get(StackUtils.fullStackName(stackName, stackVersion)); - - for (Map.Entry> entry : serviceConfigMap.entrySet()) { - ServiceConfigVO element = new ServiceConfigVO(); - element.setServiceName(entry.getKey()); - element.setConfigs(TypeConfigConverter.INSTANCE.fromDTO2VO(entry.getValue())); - list.add(element); - } - - return list; - } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/StackInitializer.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/StackInitializer.java deleted file mode 100644 index 3604ba7a5..000000000 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/StackInitializer.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bigtop.manager.server.stack; - -import org.apache.bigtop.manager.dao.po.StackPO; -import org.apache.bigtop.manager.dao.repository.StackDao; -import org.apache.bigtop.manager.server.model.dto.ServiceDTO; -import org.apache.bigtop.manager.server.model.dto.StackDTO; -import org.apache.bigtop.manager.server.utils.StackUtils; - -import org.springframework.boot.context.event.ApplicationStartedEvent; -import org.springframework.context.ApplicationListener; -import org.springframework.stereotype.Component; - -import lombok.extern.slf4j.Slf4j; - -import jakarta.annotation.Nonnull; -import jakarta.annotation.Resource; -import java.util.List; -import java.util.Map; - -/** - * Initialize the stack and persist to the database. - * 1. Get All stacks, Parse the stack.yaml - * 2. Check if the stack is already in the database - * 3. Persist the stack to the database - */ -@Slf4j -@Component -public class StackInitializer implements ApplicationListener { - - @Resource - private StackDao stackDao; - - @Override - public void onApplicationEvent(@Nonnull ApplicationStartedEvent event) { - log.info("Stacks is initializing..."); - - Map> stackMap = StackUtils.stackList(); - for (Map.Entry> entry : stackMap.entrySet()) { - StackDTO stackDTO = entry.getKey(); - String stackName = stackDTO.getStackName(); - String stackVersion = stackDTO.getStackVersion(); - - StackPO stackPO = stackDao.findByStackNameAndStackVersion(stackName, stackVersion); - if (stackPO == null) { - stackPO = new StackPO(); - stackPO.setStackName(stackName); - stackPO.setStackVersion(stackVersion); - - stackDao.save(stackPO); - } - } - - log.info("Stacks is initialized!"); - } -} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/StackModel.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/StackModel.java deleted file mode 100644 index 776b9ea57..000000000 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/StackModel.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bigtop.manager.server.stack.model; - -import lombok.Data; - -import jakarta.xml.bind.annotation.XmlAccessType; -import jakarta.xml.bind.annotation.XmlAccessorType; -import jakarta.xml.bind.annotation.XmlElement; - -@Data -@XmlAccessorType(XmlAccessType.FIELD) -public class StackModel { - - @XmlElement(name = "stack-name") - private String stackName; - - @XmlElement(name = "stack-version") - private String stackVersion; - - @XmlElement(name = "user-group") - private String userGroup; - - /** - * The scope of the stack, should be one of 'global' or 'cluster'. - */ - private String scope; - - private String root; - - private String packages; - - @XmlElement(name = "repo-template") - private String repoTemplate; -} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/StackMetainfoXml.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/StackMetainfoXml.java deleted file mode 100644 index 65ea70098..000000000 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/StackMetainfoXml.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bigtop.manager.server.stack.xml; - -import org.apache.bigtop.manager.server.stack.model.StackModel; - -import lombok.Data; - -import jakarta.xml.bind.annotation.XmlAccessType; -import jakarta.xml.bind.annotation.XmlAccessorType; -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlRootElement; - -@Data -@XmlRootElement(name = "metainfo") -@XmlAccessorType(XmlAccessType.FIELD) -public class StackMetainfoXml { - - @XmlElement(name = "stack") - private StackModel stack; -} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java index 18ffc3837..64e800e44 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java @@ -24,6 +24,8 @@ import org.apache.bigtop.manager.server.stack.model.PropertyModel; import org.apache.bigtop.manager.server.stack.xml.ConfigurationXml; +import org.apache.commons.lang3.StringUtils; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -53,9 +55,9 @@ public static List loadConfig(String fileName) { private static PropertyDTO getPropertyDTO(PropertyModel propertyModel) { PropertyDTO propertyDTO = new PropertyDTO(); propertyDTO.setDisplayName(propertyModel.getDisplayName()); - propertyDTO.setDesc(propertyModel.getDesc()); + propertyDTO.setDesc(StringUtils.strip(propertyModel.getDesc())); propertyDTO.setName(propertyModel.getName()); - propertyDTO.setValue(propertyModel.getValue()); + propertyDTO.setValue(StringUtils.strip(propertyModel.getValue())); if (propertyModel.getAttrs() != null) { AttrsModel attrsModel = propertyModel.getAttrs(); AttrsDTO attrsDTO = new AttrsDTO(); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackUtils.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackUtils.java index 86b7525c3..b69d5b1a1 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackUtils.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackUtils.java @@ -23,7 +23,6 @@ import org.apache.bigtop.manager.server.exception.ApiException; import org.apache.bigtop.manager.server.exception.ServerException; import org.apache.bigtop.manager.server.model.converter.ServiceConverter; -import org.apache.bigtop.manager.server.model.converter.StackConverter; import org.apache.bigtop.manager.server.model.dto.ComponentDTO; import org.apache.bigtop.manager.server.model.dto.PropertyDTO; import org.apache.bigtop.manager.server.model.dto.ServiceDTO; @@ -33,12 +32,10 @@ import org.apache.bigtop.manager.server.stack.dag.DAG; import org.apache.bigtop.manager.server.stack.dag.DagGraphEdge; import org.apache.bigtop.manager.server.stack.model.ServiceModel; -import org.apache.bigtop.manager.server.stack.model.StackModel; import org.apache.bigtop.manager.server.stack.xml.ServiceMetainfoXml; -import org.apache.bigtop.manager.server.stack.xml.StackMetainfoXml; import org.apache.commons.lang3.EnumUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.StringUtils; import lombok.AccessLevel; import lombok.NoArgsConstructor; @@ -47,14 +44,10 @@ import java.io.File; import java.net.URL; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; @Slf4j @NoArgsConstructor(access = AccessLevel.PRIVATE) @@ -62,8 +55,6 @@ public class StackUtils { private static final String ROLE_COMMAND_SPLIT = "-"; - private static final String BIGTOP_MANAGER_STACK_PATH = "bigtop.manager.stack.path"; - private static final String META_FILE = "metainfo.xml"; private static final String STACKS_FOLDER_NAME = "stacks"; @@ -76,175 +67,116 @@ public class StackUtils { private static final String DEPENDENCY_FILE_NAME = "order.json"; - private static final Map>> STACK_DEPENDENCY_MAP = new HashMap<>(); - - private static final Map>> STACK_CONFIG_MAP = new HashMap<>(); + public static final Map> SERVICE_CONFIG_MAP = new HashMap<>(); - private static final Map>> STACK_KEY_MAP = new HashMap<>(); + public static final Map> STACK_SERVICE_MAP = new HashMap<>(); - private static final Map> STACK_DAG_MAP = - new HashMap<>(); + public static final DAG DAG = new DAG<>(); - public static Map>> getStackDependencyMap() { - return Collections.unmodifiableMap(STACK_DEPENDENCY_MAP); - } - - public static Map>> getStackConfigMap() { - return Collections.unmodifiableMap(STACK_CONFIG_MAP); - } - - public static Map>> getStackKeyMap() { - return Collections.unmodifiableMap(STACK_KEY_MAP); - } - - public static Map> getStackDagMap() { - return Collections.unmodifiableMap(STACK_DAG_MAP); + static { + parseStack(); } /** - * Parse stack file to generate stack model * - * @return stack model {@link StackModel} */ - public static StackDTO parseStack(File versionFolder) { - StackMetainfoXml stackMetainfoXml = JaxbUtils.readFromPath( - versionFolder.getAbsolutePath() + File.separator + META_FILE, StackMetainfoXml.class); - return StackConverter.INSTANCE.fromModel2DTO(stackMetainfoXml.getStack()); + private static void parseStack() { + File stacksFolder = loadStacksFolder(); + File[] stackFolders = Optional.ofNullable(stacksFolder.listFiles()).orElse(new File[0]); + + for (File stackFolder : stackFolders) { + String stackName = stackFolder.getName(); + File[] versionFolders = Optional.ofNullable(stackFolder.listFiles()).orElse(new File[0]); + + for (File versionFolder : versionFolders) { + String stackVersion = versionFolder.getName(); + parseService(new StackDTO(stackName, stackVersion), versionFolder); + } + } } /** * Parse service file to generate service model * - * @param fullStackName full stack name + * @param stackDTO stackDTO * @return service model {@link ServiceModel} */ - public static List parseService(File versionFolder, String fullStackName) { - Map> mergedConfigMap = new HashMap<>(); + public static void parseService(StackDTO stackDTO, File versionFolder) { File[] files = new File(versionFolder.getAbsolutePath(), SERVICES_FOLDER_NAME).listFiles(); List services = new ArrayList<>(); if (files != null) { for (File file : files) { - log.info("service dir: {}", file); + log.info("parsing service: {}", file.getName()); - // metainfo.xml - ServiceMetainfoXml serviceMetainfoXml = - JaxbUtils.readFromPath(file.getAbsolutePath() + "/" + META_FILE, ServiceMetainfoXml.class); - ServiceModel serviceModel = serviceMetainfoXml.getService(); - ServiceDTO serviceDTO = ServiceConverter.INSTANCE.fromModel2DTO(serviceModel); + ServiceDTO serviceDTO = parseServiceMetaInfo(file); services.add(serviceDTO); - // configurations - List serviceConfigList = new ArrayList<>(); - File configFolder = new File(file.getAbsolutePath(), CONFIGURATION_FOLDER); - if (configFolder.exists()) { - for (File configFile : - Optional.ofNullable(configFolder.listFiles()).orElse(new File[0])) { - String configPath = configFile.getAbsolutePath(); - String fileExtension = configPath.substring(configPath.lastIndexOf(".") + 1); - if (fileExtension.equals(CONFIGURATION_FILE_EXTENSION)) { - String typeName = configPath.substring( - configPath.lastIndexOf(File.separator) + 1, configPath.lastIndexOf(".")); - - List properties = StackConfigUtils.loadConfig(configPath); - TypeConfigDTO typeConfigDTO = new TypeConfigDTO(); - typeConfigDTO.setTypeName(typeName); - typeConfigDTO.setProperties(properties); - serviceConfigList.add(typeConfigDTO); - } - } - } - - mergedConfigMap.put(serviceDTO.getServiceName(), serviceConfigList); - - // order.json - File dependencyFile = new File(file.getAbsolutePath(), DEPENDENCY_FILE_NAME); - if (dependencyFile.exists()) { - Map> dependencyMap = - STACK_DEPENDENCY_MAP.computeIfAbsent(fullStackName, k -> new HashMap<>()); + parseServiceConfigurations(file, serviceDTO.getServiceName()); - Map> dependencyMapByFile = JsonUtils.readFromFile(dependencyFile); - for (Map.Entry> entry : dependencyMapByFile.entrySet()) { - String blocked = entry.getKey(); - String fixedBlocked = blocked.split(ROLE_COMMAND_SPLIT)[0].toLowerCase() - + ROLE_COMMAND_SPLIT - + blocked.split(ROLE_COMMAND_SPLIT)[1]; - List blockers = entry.getValue(); - List fixedBlockers = blockers.stream() - .map(x -> x.split(ROLE_COMMAND_SPLIT)[0].toLowerCase() - + ROLE_COMMAND_SPLIT - + x.split(ROLE_COMMAND_SPLIT)[1]) - .toList(); - - dependencyMap.put(fixedBlocked, fixedBlockers); - } - } + parseDag(file); } - - STACK_CONFIG_MAP.put(fullStackName, mergedConfigMap); } - // log.info("Stack config map: {}", STACK_CONFIG_MAP); - log.info("Stack dependency map: {}", STACK_DEPENDENCY_MAP); - return services; + STACK_SERVICE_MAP.put(stackDTO, services); } - /** - * @return stack list map - */ - public static Map> stackList() { - File stacksFolder = loadStacksFolder(); - File[] stackFolders = Optional.ofNullable(stacksFolder.listFiles()).orElse(new File[0]); - Map> stackMap = new HashMap<>(); - - for (File stackFolder : stackFolders) { - String stackName = stackFolder.getName(); - File[] versionFolders = Optional.ofNullable(stackFolder.listFiles()).orElse(new File[0]); - - for (File versionFolder : versionFolders) { - String stackVersion = versionFolder.getName(); - String fullStackName = fullStackName(stackName, stackVersion); - log.info("Parsing stack: {}", fullStackName); - - checkStack(versionFolder); - StackDTO stackDTO = parseStack(versionFolder); - List services = parseService(versionFolder, fullStackName); - - stackMap.put(stackDTO, services); + private static ServiceDTO parseServiceMetaInfo(File file) { + ServiceMetainfoXml serviceMetainfoXml = + JaxbUtils.readFromPath(file.getAbsolutePath() + "/" + META_FILE, ServiceMetainfoXml.class); + ServiceModel serviceModel = serviceMetainfoXml.getService(); + ServiceDTO serviceDTO = ServiceConverter.INSTANCE.fromModel2DTO(serviceModel); + serviceDTO.setServiceDesc(StringUtils.strip(serviceDTO.getServiceDesc())); + return serviceDTO; + } - STACK_KEY_MAP.put(fullStackName, new ImmutablePair<>(stackDTO, services)); + private static void parseServiceConfigurations(File file, String serviceName) { + List typeConfigs = new ArrayList<>(); + File configFolder = new File(file.getAbsolutePath(), CONFIGURATION_FOLDER); + if (configFolder.exists()) { + for (File configFile : Optional.ofNullable(configFolder.listFiles()).orElse(new File[0])) { + String configPath = configFile.getAbsolutePath(); + String fileExtension = configPath.substring(configPath.lastIndexOf(".") + 1); + if (fileExtension.equals(CONFIGURATION_FILE_EXTENSION)) { + String typeName = configPath.substring( + configPath.lastIndexOf(File.separator) + 1, configPath.lastIndexOf(".")); + + List properties = StackConfigUtils.loadConfig(configPath); + TypeConfigDTO typeConfigDTO = new TypeConfigDTO(); + typeConfigDTO.setTypeName(typeName); + typeConfigDTO.setProperties(properties); + typeConfigs.add(typeConfigDTO); + } } } - initializeDag(); - return stackMap; + SERVICE_CONFIG_MAP.put(serviceName, typeConfigs); } - /** - * Initialize the DAG for each stack - */ - private static void initializeDag() { - for (Map.Entry>> mapEntry : - StackUtils.getStackDependencyMap().entrySet()) { - String fullStackName = mapEntry.getKey(); - DAG dag = new DAG<>(); - - for (Map.Entry> entry : mapEntry.getValue().entrySet()) { - String blocked = entry.getKey(); - List blockers = entry.getValue(); - addToDagNode(dag, blocked); + private static void parseDag(File file) { + File dependencyFile = new File(file.getAbsolutePath(), DEPENDENCY_FILE_NAME); + if (dependencyFile.exists()) { + Map> dependencyMapByFile = JsonUtils.readFromFile(dependencyFile); + for (Map.Entry> entry : dependencyMapByFile.entrySet()) { + String blocked = entry.getKey().split(ROLE_COMMAND_SPLIT)[0].toLowerCase() + + ROLE_COMMAND_SPLIT + + entry.getKey().split(ROLE_COMMAND_SPLIT)[1]; + List blockers = entry.getValue().stream() + .map(x -> x.split(ROLE_COMMAND_SPLIT)[0].toLowerCase() + + ROLE_COMMAND_SPLIT + + x.split(ROLE_COMMAND_SPLIT)[1]) + .toList(); + DAG.addNodeIfAbsent(blocked, getCommandWrapper(blocked)); for (String blocker : blockers) { - addToDagNode(dag, blocker); - dag.addEdge(blocker, blocked, new DagGraphEdge(blocker, blocked), false); + DAG.addNodeIfAbsent(blocked, getCommandWrapper(blocked)); + DAG.addEdge(blocker, blocked, new DagGraphEdge(blocker, blocked), false); } } - - STACK_DAG_MAP.put(fullStackName, dag); } } - private static void addToDagNode(DAG dag, String roleCommand) { + private static ComponentCommandWrapper getCommandWrapper(String roleCommand) { String[] split = roleCommand.split(ROLE_COMMAND_SPLIT); String role = split[0]; String command = split[1]; @@ -253,84 +185,72 @@ private static void addToDagNode(DAG> entry : STACK_SERVICE_MAP.entrySet()) { + for (ServiceDTO serviceDTO : entry.getValue()) { + if (serviceDTO.getServiceName().equals(serviceName)) { + return entry.getKey(); + } + } } - } - /** - * Generate full stack name - * - * @param stackName bigtop - * @param stackVersion 3.3.0 - * @return {stackName}-{stackVersion} eg. bigtop-3.3.0 - */ - public static String fullStackName(String stackName, String stackVersion) { - return stackName + "-" + stackVersion; + throw new ServerException("Service not found: " + serviceName); } - public static List getServiceDTOList(String stackName, String stackVersion) { - Map>> stackKeyMap = StackUtils.getStackKeyMap(); - ImmutablePair> immutablePair = - stackKeyMap.get(StackUtils.fullStackName(stackName, stackVersion)); - return immutablePair.getRight().stream().toList(); - } + public static List getServiceDTOList(StackDTO stackDTO) { + List serviceDTOList = STACK_SERVICE_MAP.get(stackDTO); + if (serviceDTOList == null) { + throw new ServerException("Stack not found: " + stackDTO); + } - public static ServiceDTO getServiceDTO(String stackName, String stackVersion, String serviceName) { - Map serviceNameToDTO = getServiceDTOList(stackName, stackVersion).stream() - .collect(Collectors.toMap(ServiceDTO::getServiceName, Function.identity())); - return serviceNameToDTO.get(serviceName); + return serviceDTOList; } - public static List getComponentDTOList(String stackName, String stackVersion) { - Map>> stackKeyMap = StackUtils.getStackKeyMap(); - ImmutablePair> immutablePair = - stackKeyMap.get(StackUtils.fullStackName(stackName, stackVersion)); - return immutablePair.getRight().stream() - .flatMap(serviceDTO -> serviceDTO.getComponents().stream()) - .toList(); + public static ServiceDTO getServiceDTO(String serviceName) { + for (Map.Entry> entry : STACK_SERVICE_MAP.entrySet()) { + for (ServiceDTO serviceDTO : entry.getValue()) { + if (serviceDTO.getServiceName().equals(serviceName)) { + return serviceDTO; + } + } + } + + throw new ServerException("Service not found: " + serviceName); } - public static ComponentDTO getComponentDTO(String stackName, String stackVersion, String componentName) { - Map componentNameToDTO = getComponentDTOList(stackName, stackVersion).stream() - .collect(Collectors.toMap(ComponentDTO::getComponentName, Function.identity())); - return componentNameToDTO.get(componentName); + public static ComponentDTO getComponentDTO(String componentName) { + for (Map.Entry> entry : STACK_SERVICE_MAP.entrySet()) { + for (ServiceDTO serviceDTO : entry.getValue()) { + for (ComponentDTO componentDTO : serviceDTO.getComponents()) { + if (componentDTO.getComponentName().equals(componentName)) { + return componentDTO; + } + } + } + } + + throw new ServerException("Component not found: " + componentName); } } diff --git a/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql index 740557661..8a76fcc5e 100644 --- a/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql +++ b/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql @@ -39,7 +39,7 @@ DEALLOCATE PREPARE statement; CREATE TABLE `audit_log` ( `id` BIGINT NOT NULL AUTO_INCREMENT, - `args` LONGTEXT, + `args` TEXT, `create_by` BIGINT, `create_time` DATETIME, `operation_desc` VARCHAR(255), @@ -84,10 +84,8 @@ CREATE TABLE `cluster` `state` VARCHAR(255), `update_by` BIGINT, `user_group` VARCHAR(255), - `stack_id` BIGINT, PRIMARY KEY (`id`), - UNIQUE KEY `uk_cluster_name` (`cluster_name`), - KEY `idx_cluster_stack_id` (`stack_id`) + UNIQUE KEY `uk_cluster_name` (`cluster_name`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE `component` @@ -98,7 +96,7 @@ CREATE TABLE `component` `component_name` VARCHAR(255), `create_by` BIGINT, `create_time` DATETIME, - `custom_commands` LONGTEXT, + `custom_commands` TEXT, `display_name` VARCHAR(255), `quick_link` VARCHAR(255), `cardinality` VARCHAR(255), @@ -175,26 +173,13 @@ CREATE TABLE `repo` PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -CREATE TABLE `stack` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `stack_name` VARCHAR(32) NOT NULL, - `stack_version` VARCHAR(32) NOT NULL, - `create_time` DATETIME DEFAULT NULL, - `update_time` DATETIME DEFAULT NULL, - `create_by` BIGINT, - `update_by` BIGINT, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_stack` (`stack_name`, `stack_version`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; - CREATE TABLE `task` ( `id` BIGINT NOT NULL AUTO_INCREMENT, `command` VARCHAR(255), `component_name` VARCHAR(255), - `content` LONGTEXT, - `context` LONGTEXT NOT NULL, + `content` TEXT, + `context` TEXT NOT NULL, `create_by` BIGINT, `create_time` DATETIME, `custom_command` VARCHAR(255), @@ -221,7 +206,7 @@ CREATE TABLE `job` `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, `cluster_id` BIGINT(20) UNSIGNED DEFAULT NULL, `state` VARCHAR(32) NOT NULL, - `context` LONGTEXT NOT NULL, + `context` TEXT NOT NULL, `create_time` DATETIME DEFAULT NULL, `update_time` DATETIME DEFAULT NULL, `create_by` BIGINT, @@ -236,7 +221,7 @@ CREATE TABLE `type_config` `id` BIGINT NOT NULL AUTO_INCREMENT, `create_by` BIGINT, `create_time` DATETIME, - `properties_json` LONGTEXT, + `properties_json` TEXT, `type_name` VARCHAR(255), `update_by` BIGINT, `update_time` DATETIME, @@ -284,7 +269,7 @@ CREATE TABLE `service_config` CREATE TABLE `setting` ( `id` BIGINT NOT NULL AUTO_INCREMENT, - `config_data` LONGTEXT, + `config_data` TEXT, `create_by` BIGINT, `create_time` DATETIME, `type_name` VARCHAR(255), @@ -303,7 +288,7 @@ CREATE TABLE `stage` `create_time` DATETIME DEFAULT NULL, `update_time` DATETIME DEFAULT NULL, `component_name` VARCHAR(255), - `context` LONGTEXT, + `context` TEXT, `create_by` BIGINT, `order` INTEGER, `service_name` VARCHAR(255), diff --git a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql index acc63f395..a0e53b297 100644 --- a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql +++ b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql @@ -67,7 +67,6 @@ CREATE TABLE cluster state VARCHAR(255), update_by BIGINT, user_group VARCHAR(255), - stack_id BIGINT, PRIMARY KEY (id), CONSTRAINT uk_cluster_name UNIQUE (cluster_name) ); @@ -78,7 +77,6 @@ COMMENT ON COLUMN cluster.cluster_type IS '1-Physical Machine, 2-Kubernetes'; COMMENT ON COLUMN cluster.selected IS '0-Disable, 1-Enable'; DROP INDEX IF EXISTS idx_cluster_stack_id; -CREATE INDEX idx_cluster_stack_id ON cluster (stack_id); CREATE TABLE component ( @@ -178,19 +176,6 @@ CREATE TABLE repo COMMENT ON COLUMN repo.type IS '1-services, 2-tools'; -CREATE TABLE stack -( - id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, - stack_name VARCHAR(32) NOT NULL, - stack_version VARCHAR(32) NOT NULL, - create_time TIMESTAMP(0) DEFAULT NULL, - update_time TIMESTAMP(0) DEFAULT NULL, - create_by BIGINT, - update_by BIGINT, - PRIMARY KEY (id), - CONSTRAINT uk_stack UNIQUE (stack_name, stack_version) -); - CREATE TABLE task ( id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml deleted file mode 100644 index 810a82c24..000000000 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml +++ /dev/null @@ -1,61 +0,0 @@ - - - - - - bigtop - 3.3.0 - hadoop - cluster - /opt - curl,wget - -baseurl=${baseUrl} -<#else> -mirrorlist=${mirrorList} - -path=/ -enabled=1 -gpgcheck=0 -]]> - - - - Bigtop-3.3.0 - Bigtop-3.3.0 for RockyLinux-8 x86_64 - binary - rocky8 - x86_64 - http://123.56.2.244/bigtop/3.3.0/rocky8/x86_64/ - - - Bigtop-3.3.0 - Bigtop-3.3.0 for openEuler-22.03 x86_64 - binary - openeuler22 - x86_64 - http://123.56.2.244/bigtop/3.3.0/openeuler22/x86_64/ - - - - \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/metainfo.xml deleted file mode 100644 index a80aed4b5..000000000 --- a/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/metainfo.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - - infra - 1.0.0 - hadoop - global - /opt - curl,wget - - - - Infra-1.0.0 - Infra-1.0.0 for RockyLinux-8 x86_64 - tarball - rocky8 - x86_64 - http://123.56.2.244/infra/1.0.0/tarballs/ - - - Infra-1.0.0 - Infra-1.0.0 for openEuler-22.03 aarch64 - tarball - openeuler22 - aarch64 - http://127.0.0.1/infra/1.0.0/tarballs - - - - \ No newline at end of file diff --git a/bigtop-manager-server/src/test/java/org/apache/bigtop/manager/server/controller/StackControllerTest.java b/bigtop-manager-server/src/test/java/org/apache/bigtop/manager/server/controller/StackControllerTest.java index 60b072f40..99a333b66 100644 --- a/bigtop-manager-server/src/test/java/org/apache/bigtop/manager/server/controller/StackControllerTest.java +++ b/bigtop-manager-server/src/test/java/org/apache/bigtop/manager/server/controller/StackControllerTest.java @@ -18,8 +18,6 @@ */ package org.apache.bigtop.manager.server.controller; -import org.apache.bigtop.manager.server.model.vo.ServiceComponentVO; -import org.apache.bigtop.manager.server.model.vo.ServiceConfigVO; import org.apache.bigtop.manager.server.model.vo.StackVO; import org.apache.bigtop.manager.server.service.StackService; import org.apache.bigtop.manager.server.utils.MessageSourceUtils; @@ -75,30 +73,4 @@ void listReturnsAllStacks() { assertTrue(response.isSuccess()); assertEquals(stacks, response.getData()); } - - @Test - void componentsReturnsAllComponentsForValidStack() { - String stackName = "bigtop"; - String stackVersion = "1.0.0"; - List components = Arrays.asList(new ServiceComponentVO(), new ServiceComponentVO()); - when(stackService.components(stackName, stackVersion)).thenReturn(components); - - ResponseEntity> response = stackController.components(stackName, stackVersion); - - assertTrue(response.isSuccess()); - assertEquals(components, response.getData()); - } - - @Test - void configurationsReturnsAllConfigurationsForValidStack() { - String stackName = "bigtop"; - String stackVersion = "1.0.0"; - List configurations = Arrays.asList(new ServiceConfigVO(), new ServiceConfigVO()); - when(stackService.configurations(stackName, stackVersion)).thenReturn(configurations); - - ResponseEntity> response = stackController.configurations(stackName, stackVersion); - - assertTrue(response.isSuccess()); - assertEquals(configurations, response.getData()); - } } diff --git a/bigtop-manager-server/src/test/java/org/apache/bigtop/manager/server/service/ClusterServiceTest.java b/bigtop-manager-server/src/test/java/org/apache/bigtop/manager/server/service/ClusterServiceTest.java index 42535ed00..7abda7257 100644 --- a/bigtop-manager-server/src/test/java/org/apache/bigtop/manager/server/service/ClusterServiceTest.java +++ b/bigtop-manager-server/src/test/java/org/apache/bigtop/manager/server/service/ClusterServiceTest.java @@ -20,11 +20,8 @@ package org.apache.bigtop.manager.server.service; import org.apache.bigtop.manager.dao.po.ClusterPO; -import org.apache.bigtop.manager.dao.po.RepoPO; -import org.apache.bigtop.manager.dao.po.StackPO; import org.apache.bigtop.manager.dao.repository.ClusterDao; import org.apache.bigtop.manager.dao.repository.RepoDao; -import org.apache.bigtop.manager.dao.repository.StackDao; import org.apache.bigtop.manager.server.enums.ApiExceptionEnum; import org.apache.bigtop.manager.server.exception.ApiException; import org.apache.bigtop.manager.server.model.dto.ClusterDTO; @@ -32,7 +29,6 @@ import org.apache.bigtop.manager.server.model.dto.ServiceDTO; import org.apache.bigtop.manager.server.model.dto.StackDTO; import org.apache.bigtop.manager.server.service.impl.ClusterServiceImpl; -import org.apache.bigtop.manager.server.utils.StackUtils; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -41,20 +37,15 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoExtension; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.CALLS_REAL_METHODS; -import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) @@ -64,9 +55,6 @@ public class ClusterServiceTest { @Mock private ClusterDao clusterDao; - @Mock - private StackDao stackDao; - @Mock private RepoDao repoDao; @@ -93,11 +81,8 @@ public void setup() { RepoDTO repoDTO = new RepoDTO(); repoDTO.setArch("x86_64"); clusterDTO.setRepoInfoList(List.of(repoDTO)); - StackDTO stackDTO = new StackDTO(); + StackDTO stackDTO = new StackDTO(clusterDTO.getStackName(), clusterDTO.getStackVersion()); stackDTO.setStackName("TestStack"); - mockStackKeyMap.put( - StackUtils.fullStackName(clusterDTO.getStackName(), clusterDTO.getStackVersion()), - new ImmutablePair<>(stackDTO, new ArrayList<>())); } @Test @@ -116,19 +101,4 @@ public void testListAndGetAndUpdate() { clusterDTO.setClusterName(CLUSTER_NAME); assert clusterService.update(1L, clusterDTO).getClusterName().equals(CLUSTER_NAME); } - - @Test - public void testSave() { - when(stackDao.findByStackNameAndStackVersion(any(), any())).thenReturn(new StackPO()); - when(hostService.batchSave(any(), any())).thenReturn(null); - RepoPO repoPO = new RepoPO(); - repoPO.setArch("x86_64"); - when(repoDao.findAllByClusterId(any())).thenReturn(List.of(repoPO)); - try (MockedStatic mockedStackUtils = mockStatic(StackUtils.class, CALLS_REAL_METHODS)) { - mockedStackUtils.when(StackUtils::getStackKeyMap).thenReturn(mockStackKeyMap); - clusterService.save(clusterDTO); - when(clusterDao.findByClusterName(any())).thenReturn(Optional.ofNullable(clusterPO)); - clusterService.save(clusterDTO); - } - } } From d273d0ecb481c3c97380f45ed06c5de177c7e1fe Mon Sep 17 00:00:00 2001 From: lhpqaq <657407891@qq.com> Date: Fri, 1 Nov 2024 14:30:45 +0800 Subject: [PATCH 6/6] replace prometheus --- .../stacks/infra/1.0.0/services/prometheus/metainfo.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/prometheus/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/prometheus/metainfo.xml index 7c067cca9..ec1ec1e9c 100644 --- a/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/prometheus/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/prometheus/metainfo.xml @@ -70,8 +70,8 @@ - prometheus-2.55.0-rc.0.linux-arm64.tar.gz - SHA-256:a25d8fae86ff1acd95e70405d1227142d1a8a214767c2db26a2289c97b80ec2b + prometheus-2.54.0.linux-arm64.tar.gz + SHA-256:ed50b67cb833a225ec2a53b487c6e20372b20e56dce226423fa8611c8aa50392