From 5e1d6d32a77804d25de70ecad6e26a944aca3454 Mon Sep 17 00:00:00 2001 From: Zhiguo Wu Date: Wed, 7 Aug 2024 17:53:14 +0800 Subject: [PATCH] BIGTOP-4131: Add Tez component on Bigtop-3.3.0 stack --- .../src/main/resources/bin/start.sh | 29 ++ .../command/job/AbstractServiceJob.java | 19 + .../command/job/ServiceConfigureJob.java | 8 + .../server/command/job/ServiceInstallJob.java | 3 + .../stage/ComponentConfigureStage.java | 39 ++ .../command/task/ComponentConfigureTask.java | 38 ++ .../server/stack/xml/ConfigurationXml.java | 3 - .../server/stack/xml/ServiceMetainfoXml.java | 4 - .../server/stack/xml/StackMetainfoXml.java | 3 - .../src/main/resources/bin/start.sh | 1 - .../stacks/bigtop/3.3.0/metainfo.xml | 1 - .../bigtop/3.3.0/services/hdfs/metainfo.xml | 3 +- .../bigtop/3.3.0/services/kafka/metainfo.xml | 1 - .../bigtop/3.3.0/services/kafka/order.json | 3 - .../services/tez/configuration/tez-env.xml | 54 +++ .../services/tez/configuration/tez-site.xml | 355 ++++++++++++++++++ .../bigtop/3.3.0/services/tez/metainfo.xml | 63 ++++ .../services/yarn/configuration/yarn-site.xml | 9 +- .../bigtop/3.3.0/services/yarn/metainfo.xml | 1 - .../bigtop/3.3.0/services/yarn/order.json | 8 +- .../3.3.0/services/zookeeper/metainfo.xml | 1 - .../3.3.0/services/zookeeper/order.json | 4 +- .../resources/stacks/nop/1.0.0/metainfo.xml | 1 - .../nop/1.0.0/services/kafka/metainfo.xml | 1 - .../nop/1.0.0/services/zookeeper/metainfo.xml | 1 - .../bigtop-manager-stack-bigtop/pom.xml | 53 +++ .../manager/stack/bigtop/utils/HdfsUtil.java | 84 +++++ .../v3_3_0/kafka/KafkaBrokerScript.java | 63 +--- .../stack/bigtop/v3_3_0/kafka/KafkaSetup.java | 98 +++++ .../bigtop/v3_3_0/tez/TezClientScript.java | 43 +++ .../stack/bigtop/v3_3_0/tez/TezParams.java | 101 +++++ .../stack/bigtop/v3_3_0/tez/TezSetup.java | 74 ++++ .../stack/bigtop/v3_3_0/yarn/YarnSetup.java | 3 +- .../common/utils/linux/LinuxFileUtils.java | 39 +- bigtop-manager-stack/pom.xml | 1 + dev-support/docker/rocky8/build-containers.sh | 4 +- 36 files changed, 1115 insertions(+), 101 deletions(-) create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentConfigureStage.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentConfigureTask.java create mode 100644 bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-env.xml create mode 100644 bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-site.xml create mode 100644 bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml create mode 100644 bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/utils/HdfsUtil.java create mode 100644 bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaSetup.java create mode 100644 bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java create mode 100644 bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java create mode 100644 bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java diff --git a/bigtop-manager-agent/src/main/resources/bin/start.sh b/bigtop-manager-agent/src/main/resources/bin/start.sh index 1a6f9cc61..638b5b133 100755 --- a/bigtop-manager-agent/src/main/resources/bin/start.sh +++ b/bigtop-manager-agent/src/main/resources/bin/start.sh @@ -23,12 +23,41 @@ BIGTOP_MANAGER_HOME=${BIGTOP_MANAGER_HOME:-$(cd $BIN_DIR/..; pwd)} source "${BIGTOP_MANAGER_HOME}/bin/env.sh" +usage() { + echo "usage: $PROG [--debug]" + echo " --debug - enable debug mode" + echo " -h, --help" + exit 1 +} + +DEBUG="false" +DOCKER="false" + +while [ $# -gt 0 ]; do + case "$1" in + --debug) + echo "enable debug mode." + DEBUG="true" + shift;; + -h|--help) + usage + shift;; + *) + echo "Unknown argument: '$1'" 1>&2 + usage;; + esac +done + JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms4g -Xmx4g -Xmn2g -XX:+IgnoreUnrecognizedVMOptions -XX:+PrintGCDateStamps -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} if [[ "$DOCKER" == "true" ]]; then JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport" fi +if [[ "$DEBUG" == "true" ]]; then + JAVA_OPTS="${JAVA_OPTS} -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006" +fi + cd $BIGTOP_MANAGER_HOME $JAVA_HOME/bin/java $JAVA_OPTS \ diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java index 1315077ed..4a51ba49c 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java @@ -28,6 +28,7 @@ import org.apache.bigtop.manager.dao.repository.HostComponentRepository; import org.apache.bigtop.manager.server.command.stage.CacheFileUpdateStage; import org.apache.bigtop.manager.server.command.stage.ComponentCheckStage; +import org.apache.bigtop.manager.server.command.stage.ComponentConfigureStage; import org.apache.bigtop.manager.server.command.stage.ComponentInstallStage; import org.apache.bigtop.manager.server.command.stage.ComponentStartStage; import org.apache.bigtop.manager.server.command.stage.ComponentStopStage; @@ -35,6 +36,7 @@ import org.apache.bigtop.manager.server.exception.ServerException; import org.apache.bigtop.manager.server.holder.SpringContextHolder; import org.apache.bigtop.manager.server.model.dto.ComponentDTO; +import org.apache.bigtop.manager.server.model.dto.ComponentHostDTO; import org.apache.bigtop.manager.server.model.dto.ServiceDTO; import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO; import org.apache.bigtop.manager.server.stack.dag.ComponentCommandWrapper; @@ -178,6 +180,23 @@ protected void createInstallStages() { } } + protected void createConfigureStages() { + for (ServiceCommandDTO serviceCommand : jobContext.getCommandDTO().getServiceCommands()) { + if (serviceCommand.getInstalled()) { + continue; + } + + for (ComponentHostDTO componentHost : serviceCommand.getComponentHosts()) { + String serviceName = serviceCommand.getServiceName(); + String componentName = componentHost.getComponentName(); + List hostnames = componentHost.getHostnames(); + + StageContext stageContext = createStageContext(serviceName, componentName, hostnames); + stages.add(new ComponentConfigureStage(stageContext)); + } + } + } + protected void createStartStages() { List todoList = getTodoListForCommand(Command.START); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java index 9dad58aca..6f60fc6f6 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java @@ -26,7 +26,15 @@ public ServiceConfigureJob(JobContext jobContext) { @Override protected void createStages() { + // Update cache files super.createCacheStage(); + + // Configure services + super.createConfigureStages(); + + // Restart services + super.createStopStages(); + super.createStartStages(); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceInstallJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceInstallJob.java index d97b20944..dd5c727a9 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceInstallJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceInstallJob.java @@ -67,6 +67,9 @@ protected void createStages() { // Update cache files after installed super.createCacheStage(); + // Configure services + super.createConfigureStages(); + // Start all master components super.createStartStages(); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentConfigureStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentConfigureStage.java new file mode 100644 index 000000000..08eea2ad5 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentConfigureStage.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.server.command.stage; + +import org.apache.bigtop.manager.server.command.task.ComponentConfigureTask; +import org.apache.bigtop.manager.server.command.task.Task; + +public class ComponentConfigureStage extends AbstractComponentStage { + + public ComponentConfigureStage(StageContext stageContext) { + super(stageContext); + } + + @Override + protected Task createTask(String hostname) { + return new ComponentConfigureTask(createTaskContext(hostname)); + } + + @Override + public String getName() { + return "Configure " + stageContext.getComponentDTO().getDisplayName(); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentConfigureTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentConfigureTask.java new file mode 100644 index 000000000..553d35ab8 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentConfigureTask.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.server.command.task; + +import org.apache.bigtop.manager.common.enums.Command; + +public class ComponentConfigureTask extends AbstractComponentTask { + + public ComponentConfigureTask(TaskContext taskContext) { + super(taskContext); + } + + @Override + protected Command getCommand() { + return Command.CONFIGURE; + } + + @Override + public String getName() { + return "Configure " + taskContext.getComponentDisplayName() + " on " + taskContext.getHostname(); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ConfigurationXml.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ConfigurationXml.java index 07f6a106c..7981699d9 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ConfigurationXml.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ConfigurationXml.java @@ -33,9 +33,6 @@ @XmlAccessorType(XmlAccessType.FIELD) public class ConfigurationXml { - @XmlElement(name = "schema-version") - private String schemaVersion; - @XmlElement(name = "property") private List propertyModels; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ServiceMetainfoXml.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ServiceMetainfoXml.java index cd8fb560f..b8c352e54 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ServiceMetainfoXml.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ServiceMetainfoXml.java @@ -24,7 +24,6 @@ import jakarta.xml.bind.annotation.XmlAccessType; import jakarta.xml.bind.annotation.XmlAccessorType; -import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlRootElement; @Data @@ -32,8 +31,5 @@ @XmlAccessorType(XmlAccessType.FIELD) public class ServiceMetainfoXml { - @XmlElement(name = "schema-version") - private String schemaVersion; - private ServiceModel service; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/StackMetainfoXml.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/StackMetainfoXml.java index 341589b7f..65ea70098 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/StackMetainfoXml.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/StackMetainfoXml.java @@ -32,9 +32,6 @@ @XmlAccessorType(XmlAccessType.FIELD) public class StackMetainfoXml { - @XmlElement(name = "schema-version") - private String schemaVersion; - @XmlElement(name = "stack") private StackModel stack; } diff --git a/bigtop-manager-server/src/main/resources/bin/start.sh b/bigtop-manager-server/src/main/resources/bin/start.sh index 855eea8f6..7769d50bf 100755 --- a/bigtop-manager-server/src/main/resources/bin/start.sh +++ b/bigtop-manager-server/src/main/resources/bin/start.sh @@ -58,7 +58,6 @@ if [[ "$DEBUG" == "true" ]]; then JAVA_OPTS="${JAVA_OPTS} -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005" fi - cd $BIGTOP_MANAGER_HOME $JAVA_HOME/bin/java $JAVA_OPTS \ diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml index 78099df7b..ea1435937 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml @@ -19,7 +19,6 @@ --> - 2.0 bigtop 3.3.0 diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml index c2fc51883..8bf21ddab 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml @@ -19,7 +19,6 @@ --> - 2.0 hdfs HDFS @@ -113,7 +112,7 @@ hadoop_3_3_0 hadoop_3_3_0-client hadoop_3_3_0-libhdfs - + c diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml index 5e379f3b8..844ba3ee9 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml @@ -19,7 +19,6 @@ --> - 2.0 kafka Kafka diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/order.json b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/order.json index 78adc63de..09585eb14 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/order.json +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/order.json @@ -7,8 +7,5 @@ ], "KAFKA_BROKER-RESTART": [ "ZOOKEEPER_SERVER-RESTART" - ], - "ZOOKEEPER_SERVER-STOP": [ - "KAFKA_BROKER-STOP" ] } \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-env.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-env.xml new file mode 100644 index 000000000..0f9c9ffab --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-env.xml @@ -0,0 +1,54 @@ + + + + + + + content + tez-env template + This is the freemarker template for tez-env.sh file + + + + longtext + + + + enable_heap_dump + false + Enable or disable taking Heap Dump. (true/false) + Enable heap dump + + + heap_dump_location + /tmp + Location for heap dump file + Heap dump location + + diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-site.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-site.xml new file mode 100644 index 000000000..9f6417f4f --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-site.xml @@ -0,0 +1,355 @@ + + + + + + tez.lib.uris.classpath + TEZ Lib URIs Classpath + ${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/*,${hadoop_hdfs_home!}/*,${hadoop_hdfs_home!}/lib/*,${hadoop_yarn_home!}/*,${hadoop_yarn_home!}/lib/*,${tez_home!}/*,${tez_home!}/lib/*,${tez_conf_dir!} + + Comma-delimited list of the location of the Tez libraries Classpath which will be localized for DAGs. + + + + tez.lib.uris + TEZ Lib URIs + ${tez_lib_uris!} + Comma-delimited list of the location of the Tez libraries which will be localized for DAGs. + Specifying a single .tar.gz or .tgz assumes that a compressed version of the tez libs is being used. This is + uncompressed into a tezlibs directory when running containers, and tezlibs/;tezlibs/lib/ are added to the + classpath (after . and .*). + If multiple files are specified - files are localized as regular files, contents of directories are + localized as regular files (non-recursive). + + + + tez.cluster.additional.classpath.prefix + /etc/hadoop/conf/secure + + + + tez.am.log.level + INFO + Root Logging level passed to the Tez app master + + + tez.generate.debug.artifacts + false + Generate debug artifacts such as a text representation of the submitted DAG plan + + + tez.staging-dir + TEZ Staging directory + /tmp/${tez_user!}/staging + The staging dir used while submitting DAGs + + + tez.am.resource.memory.mb + 2048 + The amount of memory to be used by the AppMaster. + Used only if the value is not specified explicitly by the DAG definition. + + + + tez.am.launch.cmd-opts + -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB ${heap_dump_opts!} + Java options for the Tez AppMaster process. The Xmx value is derived based on + tez.am.resource.memory.mb and is 80% of the value by default. + Used only if the value is not specified explicitly by the DAG definition. + + + + tez.am.launch.cluster-default.cmd-opts + -server -Djava.net.preferIPv4Stack=true + Cluster default Java options for the Tez AppMaster process. These will be prepended to the + properties specified via tez.am.launch.cmd-opts + + + + tez.am.launch.env + LD_LIBRARY_PATH=${hadoop_home!}/lib/native + + Additional execution environment entries for tez. This is not an additive property. You must preserve the + original value if + you want to have access to native libraries. + Used only if the value is not specified explicitly by the DAG definition. + + + + tez.task.resource.memory.mb + 1536 + The amount of memory to be used by launched tasks. + Used only if the value is not specified explicitly by the DAG definition. + + + + tez.task.launch.cmd-opts + -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB ${heap_dump_opts!} + Java options for tasks. The Xmx value is derived based on tez.task.resource.memory.mb and is 80% of + this value by default. + Used only if the value is not specified explicitly by the DAG definition. + + + + tez.task.launch.cluster-default.cmd-opts + -server -Djava.net.preferIPv4Stack=true + Cluster default Java options for tasks. These will be prepended to the properties specified via + tez.task.launch.cmd-opts + + + + tez.task.launch.env + LD_LIBRARY_PATH=${hadoop_home!}/lib/native + + Additional execution environment entries for tez. This is not an additive property. You must preserve the + original value if + you want to have access to native libraries. + Used only if the value is not specified explicitly by the DAG definition. + + + + tez.shuffle-vertex-manager.min-src-fraction + 0.2 + In case of a ScatterGather connection, the fraction of source tasks which should + complete before tasks for the current vertex are schedule + + + + tez.shuffle-vertex-manager.max-src-fraction + 0.4 + In case of a ScatterGather connection, once this fraction of source tasks have + completed, all tasks on the current vertex can be scheduled. Number of tasks ready for + scheduling on the current vertex scales linearly between min-fraction and max-fraction + + + + tez.am.am-rm.heartbeat.interval-ms.max + 250 + The maximum heartbeat interval between the AM and RM in milliseconds + + + tez.grouping.split-waves + 1.7 + The multiplier for available queue capacity when determining number of tasks for + a Vertex. 1.7 with 100% queue available implies generating a number of tasks roughly equal + to 170% of the available containers on the queue + + + + tez.grouping.min-size + 16777216 + Lower bound on the size (in bytes) of a grouped split, to avoid generating + too many splits + + + + tez.grouping.max-size + 1073741824 + Upper bound on the size (in bytes) of a grouped split, to avoid generating + excessively large split + + + + tez.am.container.reuse.enabled + true + Configuration to specify whether container should be reused + + + tez.am.container.reuse.rack-fallback.enabled + true + Whether to reuse containers for rack local tasks. Active only if reuse is enabled + + + + tez.am.container.reuse.non-local-fallback.enabled + false + Whether to reuse containers for non-local tasks. Active only if reuse is enabled + + + + tez.am.container.idle.release-timeout-min.millis + 10000 + The minimum amount of time to hold on to a container that is idle. Only active when reuse is + enabled. + + + + tez.am.container.idle.release-timeout-max.millis + 20000 + The maximum amount of time to hold on to a container if no task can be assigned to it immediately. + Only active when reuse is enabled. + + + + tez.am.container.reuse.locality.delay-allocation-millis + 250 + The amount of time to wait before assigning a container to the next level of + locality. NODE -> RACK -> NON_LOCAL + + + + tez.am.max.app.attempts + 2 + Specifies the total number of time the app master will run in case recovery is triggered + + + tez.am.maxtaskfailures.per.node + 10 + The maximum number of allowed task attempt failures on a node before it gets marked as blacklisted + + + tez.task.am.heartbeat.counter.interval-ms.max + 4000 + Time interval at which task counters are sent to the AM + + + tez.task.get-task.sleep.interval-ms.max + 200 + The maximum amount of time, in seconds, to wait before a task asks an AM for another task + + + tez.task.max-events-per-heartbeat + 500 + Maximum number of of events to fetch from the AM by the tasks in a single heartbeat. + + + tez.session.client.timeout.secs + -1 + Time (in seconds) to wait for AM to come up when trying to submit a DAG from the client + + + tez.session.am.dag.submit.timeout.secs + 300 + + Time (in seconds) for which the Tez AM should wait for a DAG to be submitted before shutting down + + + + tez.runtime.compress + true + Whether intermediate data should be compressed or not + + + tez.runtime.compress.codec + org.apache.hadoop.io.compress.SnappyCodec + + The coded to be used if compressing intermediate data. Only applicable if tez.runtime.compress is enabled + + + + tez.runtime.unordered.output.buffer.size-mb + 100 + The size of the buffer when output does not require to be sorted + + + tez.runtime.convert.user-payload.to.history-text + false + Whether to publish configuration information to History logger + + + tez.use.cluster.hadoop-libs + false + + This being true implies that the deployment is relying on hadoop jars being available on the cluster on all nodes. + + + + tez.am.tez-ui.history-url.template + __HISTORY_URL_BASE__?viewPath=%2F%23%2Ftez-app%2F__APPLICATION_ID__ + Template to generate the History URL for a particular Tez Application. + Template replaces __APPLICATION_ID__ with the actual applicationId and + __HISTORY_URL_BASE__ with the value from the tez.tez-ui.history-url.base config property + + + + tez.tez-ui.history-url.base + + + + + tez.am.view-acls + * + AM view ACLs. This allows the specified users/groups to view the status of the AM and all DAGs that + run within this AM. + Value format: Comma separated list of users, followed by whitespace, followed by a comma separated list of + groups. + + + + tez.runtime.optimize.local.fetch + true + If the shuffle input is on the local host bypass the http fetch and access the files directly. + + + + tez.task.generate.counters.per.io + true + Whether to generate counters on a per-edge basis for a Tez DAG. Helpful for in-depth analysis. + + + + tez.runtime.sorter.class + PIPELINED + Which sorter implementation to use + + + tez.runtime.pipelined.sorter.sort.threads + 2 + Tez runtime pipelined sorter sort threads + + + tez.runtime.io.sort.mb + 272 + The size of the sort buffer when output needs to be sorted + + + tez.counters.max + 10000 + The number of allowed counters for the executing DAG + + + tez.counters.max.groups + 3000 + The number of allowed counter groups for the executing DAG + + + tez.runtime.shuffle.fetch.buffer.percent + 0.6 + Fraction (0-1) of the available memory which can be used to retain shuffled data + + + tez.runtime.shuffle.memory.limit.percent + 0.25 + This property determines the maximum size of a shuffle segment + which can be fetched to memory. Fraction (0-1) of shuffle memory + (after applying tez.runtime.shuffle.fetch.buffer.percent) + + + + tez.history.logging.timeline-cache-plugin.old-num-dags-per-group + 5 + DAGs per group + + + yarn.timeline-service.enabled + false + Timeline service version we’re currently using. + + diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml new file mode 100644 index 000000000..42ef5e264 --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml @@ -0,0 +1,63 @@ + + + + + + tez + Tez + Tez is the next generation Hadoop Query Processing framework written on top of YARN. + 0.10.2-1 + tez + tez + + + + tez_client + Tez Client + client + 1+ + + org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez.TezClientScript + java + 1200 + + + + + + + + centos7 + rocky8 + + + x86_64 + + + tez_3_3_0 + + + + + + hdfs + + + diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml index 996cb2ec5..c830d4a60 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml @@ -91,14 +91,13 @@ yarn.nodemanager.resource.memory-mb 5120 - Amount of physical memory, in MB, that can be allocated - for containers. + Amount of physical memory, in MB, that can be allocated for containers. Memory allocated for all YARN containers on a node yarn.application.classpath - {{hadoop_conf_dir}},{{hadoop_home}}/*,{{hadoop_home}}/lib/*,{{hadoop_hdfs_home}}/*,{{hadoop_hdfs_home}}/lib/*,{{hadoop_yarn_home}}/*,{{hadoop_yarn_home}}/lib/* + ${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/*,${hadoop_hdfs_home!}/*,${hadoop_hdfs_home!}/lib/*,${hadoop_yarn_home!}/*,${hadoop_yarn_home!}/lib/* Classpath for typical applications. @@ -402,12 +401,12 @@ yarn.authorization-provider - Yarn authorization provider class. + Yarn authorization provider class. yarn.admin.acl yarn - ACL of who can be admin of the YARN cluster. + ACL of who can be admin of the YARN cluster. diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml index 44ea8ffe7..9e0d03d11 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml @@ -19,7 +19,6 @@ --> - 2.0 yarn YARN diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/order.json b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/order.json index 9c9addb58..ea235ec14 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/order.json +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/order.json @@ -1,16 +1,14 @@ { "RESOURCEMANAGER-START": [ - "ZOOKEEPER_SERVER-START" + "ZOOKEEPER_SERVER-START", + "NAMENODE-START", + "DATANODE-START" ], "NODEMANAGER-START": [ "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START" ], - "RESOURCEMANAGER-START": [ - "NAMENODE-START", - "DATANODE-START" - ], "RESOURCEMANAGER-RESTART": [ "NAMENODE-RESTART" ], diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml index 64f10b141..043b67e4b 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml @@ -19,7 +19,6 @@ --> - 2.0 zookeeper ZooKeeper diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/order.json b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/order.json index a0d75ea41..0da471753 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/order.json +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/order.json @@ -1,5 +1,5 @@ { - "KAFKA_BROKER-INSTALL": [ - "ZOOKEEPER_SERVER-INSTALL" + "ZOOKEEPER_SERVER-STOP": [ + "KAFKA_BROKER-STOP" ] } \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/metainfo.xml index 1c956f044..c87d7b7db 100644 --- a/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/metainfo.xml @@ -19,7 +19,6 @@ --> - 2.0 nop 1.0.0 diff --git a/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/kafka/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/kafka/metainfo.xml index cd3d23edc..dd3d0c2c6 100644 --- a/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/kafka/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/kafka/metainfo.xml @@ -19,7 +19,6 @@ --> - 2.0 kafka Kafka diff --git a/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/zookeeper/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/zookeeper/metainfo.xml index 55db55e88..1ebb84cd0 100644 --- a/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/zookeeper/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/zookeeper/metainfo.xml @@ -19,7 +19,6 @@ --> - 2.0 zookeeper ZooKeeper diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/pom.xml b/bigtop-manager-stack/bigtop-manager-stack-bigtop/pom.xml index ad85d1aeb..a9022edfd 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/pom.xml +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/pom.xml @@ -29,6 +29,11 @@ ${project.artifactId} Bigtop Manager Stack Bigtop + + + 3.3.6 + + org.apache.bigtop @@ -39,6 +44,54 @@ com.google.auto.service auto-service + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/utils/HdfsUtil.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/utils/HdfsUtil.java new file mode 100644 index 000000000..7cf6fd147 --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/utils/HdfsUtil.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.utils; + +import org.apache.bigtop.manager.stack.common.exception.StackException; +import org.apache.bigtop.manager.stack.common.utils.LocalSettings; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +import java.net.URI; +import java.security.PrivilegedAction; +import java.text.MessageFormat; +import java.util.List; + +@Data +@Slf4j +public class HdfsUtil { + + public static void uploadFile(String user, String localFilePath, String destDir) { + uploadFile(user, localFilePath, destDir, null); + } + + public static void uploadFile(String user, String localFilePath, String destDir, String destFilename) { + Configuration conf = new Configuration(); + conf.addResource(new Path("/etc/hadoop/conf/core-site.xml")); + conf.addResource(new Path("/etc/hadoop/conf/hdfs-site.xml")); + + List namenodeList = LocalSettings.hosts("namenode"); + if (CollectionUtils.isEmpty(namenodeList)) { + String msg = "No namenode found in the cluster"; + log.error(msg); + throw new StackException(msg); + } + + String hdfsUri = MessageFormat.format("hdfs://{0}:8020", namenodeList.get(0)); + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); + try { + ugi.doAs((PrivilegedAction) () -> { + try (FileSystem fs = FileSystem.get(new URI(hdfsUri), conf)) { + // Create dest dir if not exist + Path destDirPath = new Path(destDir); + if (!fs.exists(destDirPath)) { + fs.mkdirs(destDirPath); + } + + // upload file + Path destFilePath = destFilename == null ? new Path(destDir) : new Path(destDir, destFilename); + fs.copyFromLocalFile(new Path(localFilePath), destFilePath); + } catch (Exception e) { + log.error("Error while uploading file to hdfs", e); + throw new StackException(e); + } + + return null; + }); + } catch (Exception e) { + log.error("Error while uploading file to hdfs", e); + throw new StackException(e); + } + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java index e72467d07..c44c85c6a 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java @@ -21,11 +21,8 @@ import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.spi.stack.Params; import org.apache.bigtop.manager.spi.stack.Script; -import org.apache.bigtop.manager.stack.common.enums.ConfigType; import org.apache.bigtop.manager.stack.common.exception.StackException; -import org.apache.bigtop.manager.stack.common.utils.LocalSettings; import org.apache.bigtop.manager.stack.common.utils.PackageUtils; -import org.apache.bigtop.manager.stack.common.utils.linux.LinuxFileUtils; import org.apache.bigtop.manager.stack.common.utils.linux.LinuxOSUtils; import com.google.auto.service.AutoService; @@ -33,13 +30,6 @@ import java.io.IOException; import java.text.MessageFormat; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_644; -import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_755; -import static org.apache.bigtop.manager.common.constants.Constants.ROOT_USER; @Slf4j @AutoService(Script.class) @@ -52,58 +42,7 @@ public ShellResult install(Params params) { @Override public ShellResult configure(Params params) { - KafkaParams kafkaParams = (KafkaParams) params; - - String confDir = kafkaParams.confDir(); - String kafkaUser = kafkaParams.user(); - String kafkaGroup = kafkaParams.group(); - - LinuxFileUtils.createDirectories(kafkaParams.getKafkaDataDir(), kafkaUser, kafkaGroup, PERMISSION_755, true); - LinuxFileUtils.createDirectories(kafkaParams.getKafkaLogDir(), kafkaUser, kafkaGroup, PERMISSION_755, true); - LinuxFileUtils.createDirectories(kafkaParams.getKafkaPidDir(), kafkaUser, kafkaGroup, PERMISSION_755, true); - - // server.properties - List zookeeperServerHosts = LocalSettings.hosts("zookeeper_server"); - Map paramMap = new HashMap<>(); - paramMap.put("zk_server_list", zookeeperServerHosts); - paramMap.put("host", kafkaParams.hostname()); - LinuxFileUtils.toFile( - ConfigType.PROPERTIES, - MessageFormat.format("{0}/server.properties", confDir), - kafkaUser, - kafkaGroup, - PERMISSION_644, - kafkaParams.kafkaBroker(), - paramMap); - - // kafka-env - LinuxFileUtils.toFileByTemplate( - kafkaParams.getKafkaEnvContent(), - MessageFormat.format("{0}/kafka-env.sh", confDir), - kafkaUser, - kafkaGroup, - PERMISSION_644, - kafkaParams.getGlobalParamsMap()); - - // log4j - LinuxFileUtils.toFileByTemplate( - kafkaParams.getKafkaLog4jContent(), - MessageFormat.format("{0}/log4j.properties", confDir), - kafkaUser, - kafkaGroup, - PERMISSION_644, - kafkaParams.getGlobalParamsMap()); - - // kafka.limits - LinuxFileUtils.toFileByTemplate( - kafkaParams.kafkaLimits(), - MessageFormat.format("{0}/kafka.conf", KafkaParams.LIMITS_CONF_DIR), - ROOT_USER, - ROOT_USER, - PERMISSION_644, - kafkaParams.getGlobalParamsMap()); - - return ShellResult.success("Kafka Server Configure success!"); + return KafkaSetup.config(params); } @Override diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaSetup.java new file mode 100644 index 000000000..ca5c2369f --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaSetup.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.kafka; + +import org.apache.bigtop.manager.common.shell.ShellResult; +import org.apache.bigtop.manager.spi.stack.Params; +import org.apache.bigtop.manager.stack.common.enums.ConfigType; +import org.apache.bigtop.manager.stack.common.utils.LocalSettings; +import org.apache.bigtop.manager.stack.common.utils.linux.LinuxFileUtils; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.text.MessageFormat; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_644; +import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_755; +import static org.apache.bigtop.manager.common.constants.Constants.ROOT_USER; + +@Slf4j +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class KafkaSetup { + + public static ShellResult config(Params params) { + KafkaParams kafkaParams = (KafkaParams) params; + + String confDir = kafkaParams.confDir(); + String kafkaUser = kafkaParams.user(); + String kafkaGroup = kafkaParams.group(); + + LinuxFileUtils.createDirectories(kafkaParams.getKafkaDataDir(), kafkaUser, kafkaGroup, PERMISSION_755, true); + LinuxFileUtils.createDirectories(kafkaParams.getKafkaLogDir(), kafkaUser, kafkaGroup, PERMISSION_755, true); + LinuxFileUtils.createDirectories(kafkaParams.getKafkaPidDir(), kafkaUser, kafkaGroup, PERMISSION_755, true); + + // server.properties + List zookeeperServerHosts = LocalSettings.hosts("zookeeper_server"); + Map paramMap = new HashMap<>(); + paramMap.put("zk_server_list", zookeeperServerHosts); + paramMap.put("host", kafkaParams.hostname()); + LinuxFileUtils.toFile( + ConfigType.PROPERTIES, + MessageFormat.format("{0}/server.properties", confDir), + kafkaUser, + kafkaGroup, + PERMISSION_644, + kafkaParams.kafkaBroker(), + paramMap); + + // kafka-env + LinuxFileUtils.toFileByTemplate( + kafkaParams.getKafkaEnvContent(), + MessageFormat.format("{0}/kafka-env.sh", confDir), + kafkaUser, + kafkaGroup, + PERMISSION_644, + kafkaParams.getGlobalParamsMap()); + + // log4j + LinuxFileUtils.toFileByTemplate( + kafkaParams.getKafkaLog4jContent(), + MessageFormat.format("{0}/log4j.properties", confDir), + kafkaUser, + kafkaGroup, + PERMISSION_644, + kafkaParams.getGlobalParamsMap()); + + // kafka.limits + LinuxFileUtils.toFileByTemplate( + kafkaParams.kafkaLimits(), + MessageFormat.format("{0}/kafka.conf", KafkaParams.LIMITS_CONF_DIR), + ROOT_USER, + ROOT_USER, + PERMISSION_644, + kafkaParams.getGlobalParamsMap()); + + return ShellResult.success("Kafka Server Configure success!"); + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java new file mode 100644 index 000000000..648680cfd --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez; + +import org.apache.bigtop.manager.common.shell.ShellResult; +import org.apache.bigtop.manager.spi.stack.ClientScript; +import org.apache.bigtop.manager.spi.stack.Params; +import org.apache.bigtop.manager.spi.stack.Script; +import org.apache.bigtop.manager.stack.common.utils.PackageUtils; + +import com.google.auto.service.AutoService; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@AutoService(Script.class) +public class TezClientScript implements ClientScript { + + @Override + public ShellResult install(Params params) { + return PackageUtils.install(params.getPackageList()); + } + + @Override + public ShellResult configure(Params params) { + return TezSetup.config(params); + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java new file mode 100644 index 000000000..f5df822e9 --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez; + +import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; +import org.apache.bigtop.manager.stack.common.annotations.GlobalParams; +import org.apache.bigtop.manager.stack.common.utils.BaseParams; +import org.apache.bigtop.manager.stack.common.utils.LocalSettings; + +import org.apache.commons.lang3.StringUtils; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.text.MessageFormat; +import java.util.Map; + +@Getter +@Slf4j +public class TezParams extends BaseParams { + + private String headDumpOpts; + private String tezEnvContent; + + public TezParams(CommandPayload commandPayload) { + super(commandPayload); + globalParamsMap.put("tez_user", user()); + globalParamsMap.put("tez_group", group()); + globalParamsMap.put("java_home", "/usr/local/java"); + globalParamsMap.put("hadoop_home", hadoopHome()); + globalParamsMap.put("hadoop_conf_dir", hadoopConfDir()); + globalParamsMap.put("hadoop_hdfs_home", hdfsHome()); + globalParamsMap.put("hadoop_yarn_home", yarnHome()); + globalParamsMap.put("tez_home", serviceHome()); + globalParamsMap.put("tez_conf_dir", confDir()); + globalParamsMap.put("head_dump_opts", headDumpOpts); + } + + @GlobalParams + public Map tezSite() { + return LocalSettings.configurations(serviceName(), "tez-site"); + } + + @GlobalParams + public Map tezEnv() { + Map tezEnv = LocalSettings.configurations(serviceName(), "tez-env"); + + String heapDumpEnabled = (String) tezEnv.get("enable_heap_dump"); + if (StringUtils.isNotBlank(heapDumpEnabled) && Boolean.parseBoolean(heapDumpEnabled)) { + String heapDumpLocation = StringUtils.defaultIfBlank((String) tezEnv.get("heap_dump_location"), "/tmp"); + headDumpOpts = + MessageFormat.format("-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath={0}", heapDumpLocation); + } + + tezEnvContent = (String) tezEnv.get("content"); + + return tezEnv; + } + + @Override + public String confDir() { + return "/etc/tez/conf"; + } + + @Override + public String serviceHome() { + return stackLibDir() + "/tez"; + } + + public String hadoopConfDir() { + return "/etc/hadoop/conf"; + } + + public String hadoopHome() { + return stackLibDir() + "/hadoop"; + } + + public String hdfsHome() { + return stackLibDir() + "/hadoop-hdfs"; + } + + public String yarnHome() { + return stackLibDir() + "/hadoop-yarn"; + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java new file mode 100644 index 000000000..fef4f1a3f --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez; + +import org.apache.bigtop.manager.common.constants.Constants; +import org.apache.bigtop.manager.common.shell.ShellResult; +import org.apache.bigtop.manager.spi.stack.Params; +import org.apache.bigtop.manager.stack.common.enums.ConfigType; +import org.apache.bigtop.manager.stack.common.utils.linux.LinuxFileUtils; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.text.MessageFormat; + +import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_644; + +@Slf4j +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class TezSetup { + + public static ShellResult config(Params params) { + TezParams tezParams = (TezParams) params; + + String confDir = tezParams.confDir(); + String tezUser = tezParams.user(); + String tezGroup = tezParams.group(); + + // tez-site + log.info("Generating {}/tez-site.xml file", confDir); + LinuxFileUtils.toFile( + ConfigType.XML, + MessageFormat.format("{0}/tez-site.xml", confDir), + tezUser, + tezGroup, + Constants.PERMISSION_644, + tezParams.tezSite(), + tezParams.getGlobalParamsMap()); + + // tez-env + log.info("Generating {}/tez-env.sh file", confDir); + LinuxFileUtils.toFileByTemplate( + tezParams.getTezEnvContent(), + MessageFormat.format("{0}/tez-env.sh", confDir), + tezUser, + tezGroup, + PERMISSION_644, + tezParams.getGlobalParamsMap()); + + // maybe we should upload tez.tar.gz to HDFS here? + // log.info("Uploading tez.tar.gz to HDFS"); + // HdfsUtil.uploadFile(tezUser, tezParams.serviceHome() + "/tez.tar.gz", "/apps/tez"); + + log.info("Successfully configured Tez"); + return ShellResult.success("Tez Configure success!"); + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java index f11c908df..24e35b2db 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java @@ -103,7 +103,8 @@ public static ShellResult config(Params params, String componentName) { yarnUser, yarnGroup, Constants.PERMISSION_644, - yarnParams.yarnSite()); + yarnParams.yarnSite(), + yarnParams.getGlobalParamsMap()); // log4j LinuxFileUtils.toFileByTemplate( diff --git a/bigtop-manager-stack/bigtop-manager-stack-common/src/main/java/org/apache/bigtop/manager/stack/common/utils/linux/LinuxFileUtils.java b/bigtop-manager-stack/bigtop-manager-stack-common/src/main/java/org/apache/bigtop/manager/stack/common/utils/linux/LinuxFileUtils.java index a84e8c656..dff623c72 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-common/src/main/java/org/apache/bigtop/manager/stack/common/utils/linux/LinuxFileUtils.java +++ b/bigtop-manager-stack/bigtop-manager-stack-common/src/main/java/org/apache/bigtop/manager/stack/common/utils/linux/LinuxFileUtils.java @@ -22,6 +22,7 @@ import org.apache.bigtop.manager.common.utils.JsonUtils; import org.apache.bigtop.manager.common.utils.YamlUtils; import org.apache.bigtop.manager.stack.common.enums.ConfigType; +import org.apache.bigtop.manager.stack.common.exception.StackException; import org.apache.bigtop.manager.stack.common.utils.template.TemplateUtils; import org.apache.commons.lang3.StringUtils; @@ -226,7 +227,7 @@ public static void createDirectories( Path path = Paths.get(dirPath); if (Files.isSymbolicLink(path)) { - log.warn("unable to create symbolic link: [{}]", dirPath); + log.error("Directory is symbolic link: [{}]", dirPath); return; } @@ -240,4 +241,40 @@ public static void createDirectories( updateOwner(dirPath, owner, group, recursive); updatePermissions(dirPath, permissions, recursive); } + + /** + * create symbolic link + * + * @param target target file + * @param source source file + */ + public static void createSymbolicLink(String target, String source) { + if (StringUtils.isBlank(target) || StringUtils.isBlank(source)) { + log.error("target, source must not be null"); + return; + } + + Path targetPath = Paths.get(target); + Path sourcePath = Paths.get(source); + + try { + log.info("Creating symbolic link: [{}] -> [{}]", target, source); + if (Files.exists(targetPath)) { + if (Files.isSymbolicLink(targetPath)) { + Path existingSourcePath = Files.readSymbolicLink(targetPath); + if (!existingSourcePath.equals(sourcePath)) { + throw new IOException( + "Symbolic link already exists and points to a different source: " + existingSourcePath); + } + } else { + throw new IOException("Target path exists and is not a symbolic link: " + target); + } + } + + Files.createSymbolicLink(targetPath, sourcePath); + } catch (IOException e) { + log.error("Error when create symbolic link", e); + throw new StackException(e); + } + } } diff --git a/bigtop-manager-stack/pom.xml b/bigtop-manager-stack/pom.xml index 98556bb21..c9bf86373 100644 --- a/bigtop-manager-stack/pom.xml +++ b/bigtop-manager-stack/pom.xml @@ -52,6 +52,7 @@ pom import + com.google.auto.service auto-service diff --git a/dev-support/docker/rocky8/build-containers.sh b/dev-support/docker/rocky8/build-containers.sh index fda727a83..effd34f2f 100755 --- a/dev-support/docker/rocky8/build-containers.sh +++ b/dev-support/docker/rocky8/build-containers.sh @@ -50,7 +50,7 @@ echo -e "\033[32mCreating network bigtop-manager\033[0m" docker network create --driver bridge bigtop-manager echo -e "\033[32mCreating container bigtop-manager-server\033[0m" -docker run -it -d -p 13306:3306 -p 15005:5005 -p 18080:8080 --name bigtop-manager-server --hostname bigtop-manager-server --network bigtop-manager --cap-add=SYS_TIME bigtop-manager/develop:trunk-rocky-8 +docker run -it -d -p 13306:3306 -p 15005:5005 -p 15006:5006 -p 18080:8080 --name bigtop-manager-server --hostname bigtop-manager-server --network bigtop-manager --cap-add=SYS_TIME bigtop-manager/develop:trunk-rocky-8 docker cp ../../../bigtop-manager-server/target/bigtop-manager-server bigtop-manager-server:/opt/ docker cp ../../../bigtop-manager-agent/target/bigtop-manager-agent bigtop-manager-server:/opt/ SERVER_PUB_KEY=`docker exec bigtop-manager-server /bin/cat /root/.ssh/id_rsa.pub` @@ -116,7 +116,7 @@ docker exec bigtop-manager-agent-01 bash -c "sed -i 's/host: localhost/host: $BI docker exec bigtop-manager-agent-02 bash -c "sed -i 's/host: localhost/host: $BIGTOP_MANAGER_SERVER_IP/' /opt/bigtop-manager-agent/conf/application.yml" docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-server/bin/start.sh --debug > /dev/null 2>&1 &" -docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" +docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh --debug > /dev/null 2>&1 &" docker exec bigtop-manager-agent-01 bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" docker exec bigtop-manager-agent-02 bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &"