BIGTOP-4130: Add MapReduce2 component on Bigtop-3.3.0 stack (#45)

apache · Aug 16, 2024 · 2c90eaa · 2c90eaa
1 parent 42eaba5
commit 2c90eaa
Show file tree

Hide file tree

Showing 18 changed files with 882 additions and 21 deletions.
diff --git a/...src/main/java/org/apache/bigtop/manager/agent/service/ComponentStatusServiceGrpcImpl.java b/...src/main/java/org/apache/bigtop/manager/agent/service/ComponentStatusServiceGrpcImpl.java
@@ -45,6 +45,7 @@ public void getComponentStatus(
             CommandPayload commandPayload = new CommandPayload();
             commandPayload.setCommand(Command.STATUS);
             commandPayload.setServiceName(request.getServiceName());
+            commandPayload.setServiceUser(request.getServiceUser());
             commandPayload.setComponentName(request.getComponentName());
             commandPayload.setCommandScript(JsonUtils.readFromString(request.getCommandScript(), ScriptInfo.class));
             commandPayload.setRoot(request.getRoot());

diff --git a/bigtop-manager-grpc/src/main/resources/proto/component_status.proto b/bigtop-manager-grpc/src/main/resources/proto/component_status.proto
@@ -28,12 +28,13 @@ service ComponentStatusService {
 
 message ComponentStatusRequest {
   string service_name = 1;
-  string component_name = 2;
-  string command_script = 3;
+  string service_user = 2;
+  string component_name = 3;
+  string command_script = 4;
   // TODO Unnecessary, should be removed in the future
-  string root = 4;
-  string stack_name = 5;
-  string stack_version = 6;
+  string root = 5;
+  string stack_name = 6;
+  string stack_version = 7;
 }
 
 message ComponentStatusReply {

diff --git a/...server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java b/...server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java
@@ -142,6 +142,11 @@ protected Boolean isSlaveComponent(String componentName) {
         return componentDTO.getCategory().equalsIgnoreCase(ComponentCategories.SLAVE);
     }
 
+    protected Boolean isClientComponent(String componentName) {
+        ComponentDTO componentDTO = StackUtils.getComponentDTO(stackName, stackVersion, componentName);
+        return componentDTO.getCategory().equalsIgnoreCase(ComponentCategories.CLIENT);
+    }
+
     protected List<String> findHostnamesByComponentName(String componentName) {
         List<HostComponentPO> hostComponentPOList =
                 hostComponentRepository.findAllByComponentPOClusterPOIdAndComponentPOComponentName(
@@ -205,7 +210,7 @@ protected void createStartStages() {
             String componentName = split[0];
             String serviceName = findServiceNameByComponentName(componentName);
 
-            if (!(isMasterComponent(componentName) || isSlaveComponent(componentName))) {
+            if (isClientComponent(componentName)) {
                 continue;
             }
 
@@ -227,7 +232,7 @@ protected void createStopStages() {
             String componentName = split[0];
             String serviceName = findServiceNameByComponentName(componentName);
 
-            if (!(isMasterComponent(componentName) || isSlaveComponent(componentName))) {
+            if (isClientComponent(componentName)) {
                 continue;
             }
 
@@ -249,7 +254,7 @@ protected void createCheckStages() {
             String componentName = split[0];
             String serviceName = findServiceNameByComponentName(componentName);
 
-            if (!isMasterComponent(componentName)) {
+            if (isClientComponent(componentName)) {
                 continue;
             }
 

diff --git a/...er/src/main/java/org/apache/bigtop/manager/server/scheduler/ComponentStatusScheduler.java b/...er/src/main/java/org/apache/bigtop/manager/server/scheduler/ComponentStatusScheduler.java
@@ -66,6 +66,7 @@ public void execute() {
 
             ComponentStatusRequest request = ComponentStatusRequest.newBuilder()
                     .setServiceName(servicePO.getServiceName())
+                    .setServiceUser(servicePO.getServiceUser())
                     .setComponentName(componentPO.getComponentName())
                     .setCommandScript(componentPO.getCommandScript())
                     .setRoot(clusterPO.getRoot())

diff --git a/...erver/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-env.xml b/...erver/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-env.xml
@@ -0,0 +1,76 @@
+<?xml version="1.0"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+-->
+
+<configuration>
+    <property>
+        <name>mapred_log_dir_prefix</name>
+        <value>/var/log/hadoop-mapreduce</value>
+        <display-name>Mapreduce Log Dir Prefix</display-name>
+        <description>Mapreduce Log Dir Prefix</description>
+    </property>
+    <property>
+        <name>mapred_pid_dir_prefix</name>
+        <value>/var/run/hadoop-mapreduce</value>
+        <display-name>Mapreduce PID Dir Prefix</display-name>
+        <description>Mapreduce PID Dir Prefix</description>
+    </property>
+    <property>
+        <name>jobhistory_heapsize</name>
+        <display-name>History Server heap size</display-name>
+        <value>1024</value>
+        <description>Value for JobHistoryServer heap_size variable in hadoop-env.sh</description>
+    </property>
+    <property>
+        <name>mapred_user_nofile_limit</name>
+        <value>32768</value>
+        <description>Max open files limit setting for MAPREDUCE user.</description>
+    </property>
+    <property>
+        <name>mapred_user_nproc_limit</name>
+        <value>65536</value>
+        <description>Max number of processes limit setting for MAPREDUCE user.</description>
+    </property>
+    <property>
+        <name>content</name>
+        <display-name>mapred-env template</display-name>
+        <description>This is the freemarker template for mapred-env.sh file</description>
+        <value><![CDATA[
+export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=${jobhistory_heapsize!}
+
+export HADOOP_ROOT_LOGGER=INFO,RFA
+
+export HADOOP_HOME=${hadoop_home}
+export HADOOP_YARN_HOME=${hadoop_yarn_home}
+export HADOOP_MAPRED_HOME=${hadoop_mapred_home}
+
+# Could be enabled from deployment option if necessary
+export HADOOP_LOG_DIR=${mapred_log_dir_prefix!}/$USER
+export HADOOP_PID_DIR=${mapred_pid_dir_prefix!}/$USER
+export MAPRED_HISTORYSERVER_OPTS="$MAPRED_HISTORYSERVER_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
+export HADOOP_LIBEXEC_DIR=${hadoop_libexec_dir}
+export HADOOP_OPTS="$HADOOP_OPTS --add-opens java.base/java.lang=ALL-UNNAMED"
+export JAVA_HOME=${java_home}
+]]>
+        </value>
+        <attrs>
+            <type>longtext</type>
+        </attrs>
+    </property>
+</configuration>