BIGTOP-4124:Add YARN component on Bigtop-3.3.0 stack

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/exclude-nodes.xml

@@ -0,0 +1,54 @@
+<?xml version="1.0"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+-->
+
+<configuration>
+    <property>
+        <name>content</name>
+        <display-name>hdfs slaves template</display-name>
+        <description>This is the freemarker template for hdfs file</description>
+        <value><![CDATA[
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+<#if exclude_hosts?? >
+<#list exclude_hosts as host>
+${host}
+</#list>
+</#if>
+
+]]>
+        </value>
+    </property>
+</configuration>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-env.xml

@@ -0,0 +1,206 @@
+<?xml version="1.0"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+-->
+<configuration>
+    <property>
+        <name>yarn_log_dir_prefix</name>
+        <value>/var/log/hadoop-yarn</value>
+        <display-name>YARN Log Dir Prefix</display-name>
+        <description>YARN Log Dir Prefix</description>
+    </property>
+    <property>
+        <name>yarn_pid_dir_prefix</name>
+        <value>/var/run/hadoop-yarn</value>
+        <display-name>YARN PID Dir Prefix</display-name>
+        <description>YARN PID Dir Prefix</description>
+    </property>
+    <property>
+        <name>yarn_heapsize</name>
+        <value>1024</value>
+        <display-name>YARN Java heap size</display-name>
+        <description>Max heapsize for all YARN components using a numerical value in the scale of MB</description>
+    </property>
+    <property>
+        <name>resourcemanager_heapsize</name>
+        <value>1024</value>
+        <display-name>ResourceManager Java heap size</display-name>
+        <description>Max heapsize for ResourceManager using a numerical value in the scale of MB</description>
+    </property>
+    <property>
+        <name>nodemanager_heapsize</name>
+        <value>1024</value>
+        <display-name>NodeManager Java heap size</display-name>
+        <description>Max heapsize for NodeManager using a numerical value in the scale of MB</description>
+    </property>
+    <property>
+        <name>min_user_id</name>
+        <value>1000</value>
+        <display-name>Minimum user ID for submitting job</display-name>
+        <description>Set to 0 to disallow root from submitting jobs. Set to 1000 to disallow all superusers from submitting jobs</description>
+    </property>
+    <property>
+        <name>is_supported_yarn_ranger</name>
+        <value>false</value>
+        <description>Set to false by default,  needs to be set to true in stacks that use Ranger Yarn Plugin</description>
+    </property>
+    <property>
+        <name>yarn_user_nofile_limit</name>
+        <value>32768</value>
+        <description>Max open files limit setting for YARN user.</description>
+    </property>
+    <property>
+        <name>yarn_user_nproc_limit</name>
+        <value>65536</value>
+        <description>Max number of processes limit setting for YARN user.</description>
+    </property>
+    <!-- yarn-env.sh -->
+    <property>
+        <name>content</name>
+        <display-name>yarn-env template</display-name>
+        <description>This is the jinja template for yarn-env.sh file</description>
+        <value>
+            export HADOOP_YARN_HOME={{hadoop_yarn_home}}
+            USER="$(whoami)"
+            export HADOOP_LOG_DIR={{yarn_log_dir_prefix}}/$USER
+            export HADOOP_PID_DIR={{yarn_pid_dir_prefix}}/$USER
+            export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}
+            export JAVA_HOME={{java64_home}}
+            export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}"
+
+            # User for YARN daemons
+            export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
+
+            # resolve links - $0 may be a softlink
+            export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_YARN_HOME/etc/hadoop}"
+
+            # some Java parameters
+            # export JAVA_HOME=/home/y/libexec/jdk1.6.0/
+            if [ "$JAVA_HOME" != "" ]; then
+            #echo "run java in $JAVA_HOME"
+            JAVA_HOME=$JAVA_HOME
+            fi
+
+            if [ "$JAVA_HOME" = "" ]; then
+            echo "Error: JAVA_HOME is not set."
+            exit 1
+            fi
+
+            JAVA=$JAVA_HOME/bin/java
+            JAVA_HEAP_MAX=-Xmx1000m
+
+            # For setting YARN specific HEAP sizes please use this
+            # Parameter and set appropriately
+            YARN_HEAPSIZE={{yarn_heapsize}}
+
+            # check envvars which might override default args
+            if [ "$YARN_HEAPSIZE" != "" ]; then
+            JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
+            fi
+
+            # Resource Manager specific parameters
+
+            # Specify the max Heapsize for the ResourceManager using a numerical value
+            # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+            # the value to 1000.
+            # This value will be overridden by an Xmx setting specified in either HADOOP_OPTS
+            # and/or YARN_RESOURCEMANAGER_OPTS.
+            # If not specified, the default value will be picked from either YARN_HEAPMAX
+            # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+            export YARN_RESOURCEMANAGER_HEAPSIZE={{resourcemanager_heapsize}}
+
+            # Specify the JVM options to be used when starting the ResourceManager.
+            # These options will be appended to the options specified as HADOOP_OPTS
+            # and therefore may override any similar flags set in HADOOP_OPTS
+            #export YARN_RESOURCEMANAGER_OPTS=
+
+            # Node Manager specific parameters
+
+            # Specify the max Heapsize for the NodeManager using a numerical value
+            # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+            # the value to 1000.
+            # This value will be overridden by an Xmx setting specified in either HADOOP_OPTS
+            # and/or YARN_NODEMANAGER_OPTS.
+            # If not specified, the default value will be picked from either YARN_HEAPMAX
+            # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+            export YARN_NODEMANAGER_HEAPSIZE={{nodemanager_heapsize}}
+
+            # Specify the max Heapsize for the HistoryManager using a numerical value
+            # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+            # the value to 1024.
+            # This value will be overridden by an Xmx setting specified in either HADOOP_OPTS
+            # and/or YARN_HISTORYSERVER_OPTS.
+            # If not specified, the default value will be picked from either YARN_HEAPMAX
+            # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+            export YARN_HISTORYSERVER_HEAPSIZE={{apptimelineserver_heapsize}}
+
+            # Specify the JVM options to be used when starting the NodeManager.
+            # These options will be appended to the options specified as HADOOP_OPTS
+            # and therefore may override any similar flags set in HADOOP_OPTS
+            #export YARN_NODEMANAGER_OPTS=
+
+            # so that filenames w/ spaces are handled correctly in loops below
+            IFS=
+
+
+            # default log directory and file
+            if [ "$HADOOP_LOG_DIR" = "" ]; then
+            HADOOP_LOG_DIR="$HADOOP_YARN_HOME/logs"
+            fi
+            if [ "$HADOOP_LOGFILE" = "" ]; then
+            HADOOP_LOGFILE='yarn.log'
+            fi
+
+            # default policy file for service-level authorization
+            if [ "$YARN_POLICYFILE" = "" ]; then
+            YARN_POLICYFILE="hadoop-policy.xml"
+            fi
+
+            # restore ordinary behaviour
+            unset IFS
+
+
+            HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
+            HADOOP_OPTS="$HADOOP_OPTS -Dyarn.log.dir=$HADOOP_LOG_DIR"
+            HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE"
+            HADOOP_OPTS="$HADOOP_OPTS -Dyarn.log.file=$HADOOP_LOGFILE"
+            HADOOP_OPTS="$HADOOP_OPTS -Dyarn.home.dir=$HADOOP_YARN_HOME"
+            HADOOP_OPTS="$HADOOP_OPTS -Dyarn.id.str=$HADOOP_IDENT_STRING"
+            HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
+            HADOOP_OPTS="$HADOOP_OPTS -Dyarn.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
+            export YARN_NODEMANAGER_OPTS="$YARN_NODEMANAGER_OPTS -Dnm.audit.logger=INFO,NMAUDIT"
+            export YARN_RESOURCEMANAGER_OPTS="$YARN_RESOURCEMANAGER_OPTS -Drm.audit.logger=INFO,RMAUDIT"
+            if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
+            HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
+            fi
+            HADOOP_OPTS="$HADOOP_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
+            HADOOP_OPTS="$HADOOP_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}"
+
+            {% if rm_security_opts is defined %}
+            HADOOP_OPTS="{{rm_security_opts}} $HADOOP_OPTS"
+            {% endif %}
+        </value>
+    </property>
+    <property>
+        <name>service_check.queue.name</name>
+        <value>default</value>
+        <description>
+            The queue that used by service check.
+        </description>
+    </property>
+</configuration>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-log4j.xml

@@ -0,0 +1,100 @@
+<?xml version="1.0"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+-->
+<configuration>
+    <property>
+        <name>yarn_rm_summary_log_max_backup_size</name>
+        <value>256</value>
+        <description>The maximum size of backup file before the log is rotated</description>
+        <display-name>YARN Log: backup file size</display-name>
+    </property>
+    <property>
+        <name>yarn_rm_summary_log_number_of_backup_files</name>
+        <value>20</value>
+        <description>The number of backup files</description>
+        <display-name>YARN Log: # of backup files</display-name>
+    </property>
+    <property>
+        <name>content</name>
+        <display-name>yarn-log4j template</display-name>
+        <description>Custom log4j.properties</description>
+        <value>
+            #Relative to Yarn Log Dir Prefix
+            yarn.log.dir=.
+            #
+            # Job Summary Appender
+            #
+            # Use following logger to send summary to separate file defined by
+            # hadoop.mapreduce.jobsummary.log.file rolled daily:
+            # hadoop.mapreduce.jobsummary.logger=INFO,JSA
+            #
+            hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
+            hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
+            log4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender
+            # Set the ResourceManager summary log filename
+            yarn.server.resourcemanager.appsummary.log.file=hadoop-mapreduce.jobsummary.log
+            # Set the ResourceManager summary log level and appender
+            yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger}
+            #yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY
+
+            # To enable AppSummaryLogging for the RM,
+            # set yarn.server.resourcemanager.appsummary.logger to
+            # LEVEL,RMSUMMARY in hadoop-env.sh
+
+            # Appender for ResourceManager Application Summary Log
+            # Requires the following properties to be set
+            #    - hadoop.log.dir (Hadoop Log directory)
+            #    - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename)
+            #    - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender)
+            log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender
+            log4j.appender.RMSUMMARY.File=${yarn.log.dir}/${yarn.server.resourcemanager.appsummary.log.file}
+            log4j.appender.RMSUMMARY.MaxFileSize={{yarn_rm_summary_log_max_backup_size}}MB
+            log4j.appender.RMSUMMARY.MaxBackupIndex={{yarn_rm_summary_log_number_of_backup_files}}
+            log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout
+            log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+            log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
+            log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+            log4j.appender.JSA.DatePattern=.yyyy-MM-dd
+            log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
+            log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger}
+            log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false
+
+            # Audit logging for ResourceManager
+            rm.audit.logger=${hadoop.root.logger}
+            log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger=${rm.audit.logger}
+            log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger=false
+            log4j.appender.RMAUDIT=org.apache.log4j.DailyRollingFileAppender
+            log4j.appender.RMAUDIT.File=${yarn.log.dir}/rm-audit.log
+            log4j.appender.RMAUDIT.layout=org.apache.log4j.PatternLayout
+            log4j.appender.RMAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+            log4j.appender.RMAUDIT.DatePattern=.yyyy-MM-dd
+
+            # Audit logging for NodeManager
+            nm.audit.logger=${hadoop.root.logger}
+            log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger=${nm.audit.logger}
+            log4j.additivity.org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger=false
+            log4j.appender.NMAUDIT=org.apache.log4j.DailyRollingFileAppender
+            log4j.appender.NMAUDIT.File=${yarn.log.dir}/nm-audit.log
+            log4j.appender.NMAUDIT.layout=org.apache.log4j.PatternLayout
+            log4j.appender.NMAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+            log4j.appender.NMAUDIT.DatePattern=.yyyy-MM-dd
+        </value>
+    </property>
+</configuration>
+