From 2c90eaa9762caff0d8243048c02b8568f25d6808 Mon Sep 17 00:00:00 2001 From: Zhiguo Wu Date: Fri, 16 Aug 2024 10:32:40 +0800 Subject: [PATCH] BIGTOP-4130: Add MapReduce2 component on Bigtop-3.3.0 stack (#45) --- .../ComponentStatusServiceGrpcImpl.java | 1 + .../resources/proto/component_status.proto | 11 +- .../command/job/AbstractServiceJob.java | 11 +- .../scheduler/ComponentStatusScheduler.java | 1 + .../mapred/configuration/mapred-env.xml | 76 ++++ .../mapred/configuration/mapred-site.xml | 325 ++++++++++++++++++ .../mapred/configuration/mapreduce.conf.xml | 54 +++ .../bigtop/3.3.0/services/mapred/metainfo.xml | 82 +++++ .../bigtop/3.3.0/services/mapred/order.json | 4 + .../services/yarn/configuration/yarn-site.xml | 12 +- .../services/yarn/configuration/yarn.conf.xml | 4 +- .../bigtop/3.3.0/services/yarn/metainfo.xml | 2 +- .../v3_3_0/mapred/HistoryServerScript.java | 74 ++++ .../v3_3_0/mapred/MapredClientScript.java | 43 +++ .../bigtop/v3_3_0/mapred/MapredSetup.java | 94 +++++ .../v3_3_0/mapred/Mapreduce2Params.java | 99 ++++++ .../stack/bigtop/v3_3_0/tez/TezSetup.java | 4 +- .../stack/bigtop/v3_3_0/yarn/YarnSetup.java | 6 +- 18 files changed, 882 insertions(+), 21 deletions(-) create mode 100644 bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-env.xml create mode 100644 bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-site.xml create mode 100644 bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapreduce.conf.xml create mode 100644 bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/metainfo.xml create mode 100644 bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/order.json create mode 100644 bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/HistoryServerScript.java create mode 100644 bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredClientScript.java create mode 100644 bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredSetup.java create mode 100644 bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/Mapreduce2Params.java diff --git a/bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/service/ComponentStatusServiceGrpcImpl.java b/bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/service/ComponentStatusServiceGrpcImpl.java index f2cba36b..0f43995a 100644 --- a/bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/service/ComponentStatusServiceGrpcImpl.java +++ b/bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/service/ComponentStatusServiceGrpcImpl.java @@ -45,6 +45,7 @@ public void getComponentStatus( CommandPayload commandPayload = new CommandPayload(); commandPayload.setCommand(Command.STATUS); commandPayload.setServiceName(request.getServiceName()); + commandPayload.setServiceUser(request.getServiceUser()); commandPayload.setComponentName(request.getComponentName()); commandPayload.setCommandScript(JsonUtils.readFromString(request.getCommandScript(), ScriptInfo.class)); commandPayload.setRoot(request.getRoot()); diff --git a/bigtop-manager-grpc/src/main/resources/proto/component_status.proto b/bigtop-manager-grpc/src/main/resources/proto/component_status.proto index 06799da2..7431b63f 100644 --- a/bigtop-manager-grpc/src/main/resources/proto/component_status.proto +++ b/bigtop-manager-grpc/src/main/resources/proto/component_status.proto @@ -28,12 +28,13 @@ service ComponentStatusService { message ComponentStatusRequest { string service_name = 1; - string component_name = 2; - string command_script = 3; + string service_user = 2; + string component_name = 3; + string command_script = 4; // TODO Unnecessary, should be removed in the future - string root = 4; - string stack_name = 5; - string stack_version = 6; + string root = 5; + string stack_name = 6; + string stack_version = 7; } message ComponentStatusReply { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java index 4a51ba49..76bf6f5e 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java @@ -142,6 +142,11 @@ protected Boolean isSlaveComponent(String componentName) { return componentDTO.getCategory().equalsIgnoreCase(ComponentCategories.SLAVE); } + protected Boolean isClientComponent(String componentName) { + ComponentDTO componentDTO = StackUtils.getComponentDTO(stackName, stackVersion, componentName); + return componentDTO.getCategory().equalsIgnoreCase(ComponentCategories.CLIENT); + } + protected List findHostnamesByComponentName(String componentName) { List hostComponentPOList = hostComponentRepository.findAllByComponentPOClusterPOIdAndComponentPOComponentName( @@ -205,7 +210,7 @@ protected void createStartStages() { String componentName = split[0]; String serviceName = findServiceNameByComponentName(componentName); - if (!(isMasterComponent(componentName) || isSlaveComponent(componentName))) { + if (isClientComponent(componentName)) { continue; } @@ -227,7 +232,7 @@ protected void createStopStages() { String componentName = split[0]; String serviceName = findServiceNameByComponentName(componentName); - if (!(isMasterComponent(componentName) || isSlaveComponent(componentName))) { + if (isClientComponent(componentName)) { continue; } @@ -249,7 +254,7 @@ protected void createCheckStages() { String componentName = split[0]; String serviceName = findServiceNameByComponentName(componentName); - if (!isMasterComponent(componentName)) { + if (isClientComponent(componentName)) { continue; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/scheduler/ComponentStatusScheduler.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/scheduler/ComponentStatusScheduler.java index 9ef5683a..43882aba 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/scheduler/ComponentStatusScheduler.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/scheduler/ComponentStatusScheduler.java @@ -66,6 +66,7 @@ public void execute() { ComponentStatusRequest request = ComponentStatusRequest.newBuilder() .setServiceName(servicePO.getServiceName()) + .setServiceUser(servicePO.getServiceUser()) .setComponentName(componentPO.getComponentName()) .setCommandScript(componentPO.getCommandScript()) .setRoot(clusterPO.getRoot()) diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-env.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-env.xml new file mode 100644 index 00000000..ad177ca5 --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-env.xml @@ -0,0 +1,76 @@ + + + + + + mapred_log_dir_prefix + /var/log/hadoop-mapreduce + Mapreduce Log Dir Prefix + Mapreduce Log Dir Prefix + + + mapred_pid_dir_prefix + /var/run/hadoop-mapreduce + Mapreduce PID Dir Prefix + Mapreduce PID Dir Prefix + + + jobhistory_heapsize + History Server heap size + 1024 + Value for JobHistoryServer heap_size variable in hadoop-env.sh + + + mapred_user_nofile_limit + 32768 + Max open files limit setting for MAPREDUCE user. + + + mapred_user_nproc_limit + 65536 + Max number of processes limit setting for MAPREDUCE user. + + + content + mapred-env template + This is the freemarker template for mapred-env.sh file + + + + longtext + + + diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-site.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-site.xml new file mode 100644 index 00000000..2cd6f6d8 --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-site.xml @@ -0,0 +1,325 @@ + + + + + + + mapreduce.task.io.sort.mb + 512 + Sort Allocation Memory + + The total amount of buffer memory to use while sorting files, in megabytes. + By default, gives each merge stream 1MB, which should minimize seeks. + + + + mapreduce.map.sort.spill.percent + 0.7 + + The soft limit in the serialization buffer. Once reached, a thread will + begin to spill the contents to disk in the background. Note that + collection will not block if this threshold is exceeded while a spill + is already in progress, so spills may be larger than this threshold when + it is set to less than .5 + + + + mapreduce.task.io.sort.factor + 100 + + The number of streams to merge at once while sorting files. + This determines the number of open file handles. + + + + + mapreduce.cluster.administrators + hadoop + Administrators for MapReduce applications. + + + mapreduce.reduce.shuffle.parallelcopies + 30 + + The default number of parallel transfers run by reduce during the copy(shuffle) phase. + + + + mapreduce.map.speculative + false + + If true, then multiple instances of some map tasks may be executed in parallel. + + + + mapreduce.reduce.speculative + false + + If true, then multiple instances of some reduce tasks may be + executed in parallel. + + + + mapreduce.job.reduce.slowstart.completedmaps + 0.05 + + Fraction of the number of maps in the job which should be complete before reduces are scheduled for the job. + + + + mapreduce.job.counters.max + 130 + Limit on the number of counters allowed per job. + + + mapreduce.reduce.shuffle.merge.percent + 0.66 + + The usage threshold at which an in-memory merge will be + initiated, expressed as a percentage of the total memory allocated to + storing in-memory map outputs, as defined by + mapreduce.reduce.shuffle.input.buffer.percent. + + + + mapreduce.reduce.shuffle.input.buffer.percent + 0.7 + + The percentage of memory to be allocated from the maximum heap + size to storing map outputs during the shuffle. + + + + mapreduce.output.fileoutputformat.compress.type + BLOCK + + If the job outputs are to compressed as SequenceFiles, how should + they be compressed? Should be one of NONE, RECORD or BLOCK. + + + + mapreduce.reduce.input.buffer.percent + 0.0 + + The percentage of memory- relative to the maximum heap size- to + retain map outputs during the reduce. When the shuffle is concluded, any + remaining map outputs in memory must consume less than this threshold before + the reduce can begin. + + + + + mapreduce.map.output.compress + false + + Should the outputs of the maps be compressed before being sent across the network. Uses SequenceFile compression. + + + + mapreduce.task.timeout + 300000 + + The number of milliseconds before a task will be + terminated if it neither reads an input, writes an output, nor + updates its status string. + + + + mapreduce.map.memory.mb + 512 + Virtual memory for single Map task + Map Memory + + + mapreduce.reduce.memory.mb + 512 + Virtual memory for single Reduce task + Reduce Memory + + + mapreduce.shuffle.port + 13562 + + Default port that the ShuffleHandler will run on. + ShuffleHandler is a service run at the NodeManager to facilitate + transfers of intermediate Map outputs to requesting Reducers. + + + + mapreduce.jobhistory.intermediate-done-dir + Mapreduce JobHistory Intermediate Done directory + /apps/mapred/history/tmp + Directory where history files are written by MapReduce jobs. + + + mapreduce.jobhistory.done-dir + Mapreduce JobHistory Done directory + /apps/mapred/history/done + Directory where history files are managed by the MR JobHistory Server. + + + mapreduce.jobhistory.address + 0.0.0.0:10020 + Enter your JobHistoryServer hostname. + + + mapreduce.jobhistory.webapp.address + 0.0.0.0:19888 + Enter your JobHistoryServer hostname. + + + mapreduce.framework.name + yarn + + The runtime framework for executing MapReduce jobs. Can be one of local, classic or yarn. + + + + yarn.app.mapreduce.am.staging-dir + YARN App Mapreduce AM Staging directory + /apps/mapred/staging + The staging dir used while submitting jobs. + + + yarn.app.mapreduce.am.resource.mb + AppMaster Memory + 512 + The amount of memory the MR AppMaster needs. + + + yarn.app.mapreduce.am.command-opts + -Xmx1024m + + Java opts for the MR App Master processes. + The following symbol, if present, will be interpolated: @taskid@ is replaced + by current TaskID. Any other occurrences of '@' will go unchanged. + For example, to enable verbose gc logging to a file named for the taskid in + /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of: + -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc + + Usage of -Djava.library.path can cause programs to no longer function if + hadoop native libraries are used. These values should instead be set as part + of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and + mapreduce.reduce.env config settings. + + MR AppMaster Java Heap Size + + + yarn.app.mapreduce.am.admin-command-opts + -server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN + + Java opts for the MR App Master processes for admin purposes. + It will appears before the opts set by yarn.app.mapreduce.am.command-opts and + thus its options can be overridden user. + + Usage of -Djava.library.path can cause programs to no longer function if + hadoop native libraries are used. These values should instead be set as part + of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and + mapreduce.reduce.env config settings. + + MR AppMaster Java Heap Size + + + yarn.app.mapreduce.am.log.level + INFO + MR App Master process log level. + + + mapreduce.admin.map.child.java.opts + -server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN + This property stores Java options for map tasks. + + + mapreduce.admin.reduce.child.java.opts + -server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN + This property stores Java options for reduce tasks. + + + mapreduce.application.classpath + ${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/*,${hadoop_hdfs_home!}/*,${hadoop_hdfs_home!}/lib/*,${hadoop_yarn_home!}/*,${hadoop_yarn_home!}/lib/*,${hadoop_mapred_home!}/*,${hadoop_mapred_home!}/lib/* + CLASSPATH for MR applications. A comma-separated list of CLASSPATH entries. + + + mapreduce.am.max-attempts + 2 + + The maximum number of application attempts. It is a + application-specific setting. It should not be larger than the global number + set by resourcemanager. Otherwise, it will be override. The default number is + set to 2, to allow at least one retry for AM. + + + + mapreduce.map.java.opts + MR Map Java Heap Size + -Xmx1024m + Larger heap-size for child jvms of maps. + + + mapreduce.reduce.java.opts + MR Reduce Java Heap Size + -Xmx1024m + Larger heap-size for child jvms of reduces. + + + mapreduce.map.log.level + INFO + + The logging level for the map task. The allowed levels are: + OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL. + + + + mapreduce.reduce.log.level + INFO + + The logging level for the reduce task. The allowed levels are: + OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL. + + + + mapreduce.admin.user.env + LD_LIBRARY_PATH=${hadoop_home!}/lib/native + + Additional execution environment entries for map and reduce task processes. + This is not an additive property. You must preserve the original value if + you want your map and reduce tasks to have access to native libraries (compression, etc) + + + + mapreduce.output.fileoutputformat.compress + false + Should the job outputs be compressed? + + + mapreduce.jobhistory.http.policy + HTTP_ONLY + + This configures the HTTP endpoint for JobHistoryServer web UI. + The following values are supported: - HTTP_ONLY : Service is provided only + on http - HTTPS_ONLY : Service is provided only on https + + + + mapreduce.job.queuename + default + Queue to which a job is submitted. + + diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapreduce.conf.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapreduce.conf.xml new file mode 100644 index 00000000..52807898 --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapreduce.conf.xml @@ -0,0 +1,54 @@ + + + + + + content + mapreduce.conf template + This is the freemarker template for mapreduce file + + + + longtext + + + \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/metainfo.xml new file mode 100644 index 00000000..95d6894b --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/metainfo.xml @@ -0,0 +1,82 @@ + + + + + + mapreduce2 + MapReduce2 + Apache Hadoop NextGen MapReduce. + 3.3.6-1 + mapred + + + + history_server + History Server + master + 1 + + org.apache.bigtop.manager.stack.bigtop.v3_3_0.mapred.HistoryServerScript + java + 1200 + + + JobHistory UI + mapreduce.jobhistory.webapp.address + 19888 + mapreduce.jobhistory.webapp.https.address + 19890 + + + + mapreduce2_client + MapReduce2 Client + client + 1+ + + org.apache.bigtop.manager.stack.bigtop.v3_3_0.mapred.MapredClientScript + java + 1200 + + + + + + + + centos7 + rocky8 + + + x86_64 + + + hadoop_3_3_0-yarn + hadoop_3_3_0-hdfs + hadoop_3_3_0-mapreduce + + + + + + yarn + + + diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/order.json b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/order.json new file mode 100644 index 00000000..883fc62b --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/order.json @@ -0,0 +1,4 @@ +{ + "HISTORY_SERVER-START": ["NAMENODE-START", "DATANODE-START"], + "HISTORY_SERVER-RESTART": ["NAMENODE-RESTART"] +} \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml index 15ba17ed..653fd961 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml @@ -72,12 +72,12 @@ yarn.acl.enable false - Are acls enabled. + Are acls enabled. yarn.admin.acl - ACL of who can be admin of the YARN cluster. + ACL of who can be admin of the YARN cluster. @@ -99,7 +99,8 @@ yarn.nodemanager.vmem-pmem-ratio 2.1 - Ratio between virtual memory to physical memory when + + Ratio between virtual memory to physical memory when setting memory limits for containers. Container allocations are expressed in terms of physical memory, and virtual memory usage is allowed to exceed this allocation by this ratio. @@ -119,8 +120,9 @@ yarn.nodemanager.aux-services mapreduce_shuffle - Auxilliary services of NodeManager. A valid service name should only contain - a-zA-Z0-9_ and can not start with numbers + + Auxilliary services of NodeManager. + A valid service name should only contain a-zA-Z0-9_ and can not start with numbers diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn.conf.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn.conf.xml index 79438b71..facf11ff 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn.conf.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn.conf.xml @@ -42,8 +42,8 @@ # limitations under the License. # -${yarn_user} - nofile ${yarn_user_nofile_limit} -${yarn_group} - nproc ${yarn_user_nproc_limit} +${yarn_user} - nofile ${yarn_user_nofile_limit} +${yarn_user} - nproc ${yarn_user_nproc_limit} ]]> diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml index eeb5863c..f66e0ae3 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml @@ -22,7 +22,7 @@ yarn YARN - Apache Hadoop NextGen MapReduce (YARN) + Apache Hadoop NextGen YARN. 3.3.6-1 yarn diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/HistoryServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/HistoryServerScript.java new file mode 100644 index 00000000..58e5b39c --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/HistoryServerScript.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.mapred; + +import org.apache.bigtop.manager.common.shell.ShellResult; +import org.apache.bigtop.manager.stack.core.exception.StackException; +import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; +import org.apache.bigtop.manager.stack.core.spi.script.Script; +import org.apache.bigtop.manager.stack.core.utils.PackageUtils; +import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; + +import com.google.auto.service.AutoService; + +import java.text.MessageFormat; + +@AutoService(Script.class) +public class HistoryServerScript extends AbstractServerScript { + + @Override + public ShellResult install(Params params) { + return PackageUtils.install(params.getPackageList()); + } + + @Override + public ShellResult configure(Params params) { + return MapredSetup.config(params); + } + + @Override + public ShellResult start(Params params) { + configure(params); + Mapreduce2Params mapreduce2Params = (Mapreduce2Params) params; + String cmd = MessageFormat.format("{0} --daemon start historyserver", mapreduce2Params.mapredExec()); + try { + return LinuxOSUtils.sudoExecCmd(cmd, mapreduce2Params.user()); + } catch (Exception e) { + throw new StackException(e); + } + } + + @Override + public ShellResult stop(Params params) { + Mapreduce2Params mapreduce2Params = (Mapreduce2Params) params; + String cmd = MessageFormat.format("{0} --daemon stop historyserver", mapreduce2Params.mapredExec()); + try { + return LinuxOSUtils.sudoExecCmd(cmd, mapreduce2Params.user()); + } catch (Exception e) { + throw new StackException(e); + } + } + + @Override + public ShellResult status(Params params) { + Mapreduce2Params mapreduce2Params = (Mapreduce2Params) params; + return LinuxOSUtils.checkProcess(mapreduce2Params.getHistoryServerPidFile()); + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredClientScript.java new file mode 100644 index 00000000..48429f5c --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredClientScript.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.mapred; + +import org.apache.bigtop.manager.common.shell.ShellResult; +import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript; +import org.apache.bigtop.manager.stack.core.spi.script.Script; +import org.apache.bigtop.manager.stack.core.utils.PackageUtils; + +import com.google.auto.service.AutoService; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@AutoService(Script.class) +public class MapredClientScript extends AbstractClientScript { + + @Override + public ShellResult install(Params params) { + return PackageUtils.install(params.getPackageList()); + } + + @Override + public ShellResult configure(Params params) { + return MapredSetup.config(params); + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredSetup.java new file mode 100644 index 00000000..79ba53f7 --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredSetup.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.mapred; + +import org.apache.bigtop.manager.common.constants.Constants; +import org.apache.bigtop.manager.common.shell.ShellResult; +import org.apache.bigtop.manager.stack.bigtop.utils.HdfsUtil; +import org.apache.bigtop.manager.stack.core.enums.ConfigType; +import org.apache.bigtop.manager.stack.core.param.BaseParams; +import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.utils.LocalSettings; +import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.text.MessageFormat; + +@Slf4j +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class MapredSetup { + + public static ShellResult config(Params params) { + Mapreduce2Params mapreduce2Params = (Mapreduce2Params) params; + String hdfsUser = LocalSettings.users().get("hdfs"); + String mapredUser = params.user(); + String mapredGroup = params.group(); + String confDir = mapreduce2Params.confDir(); + + LinuxFileUtils.createDirectories( + mapreduce2Params.getMapredLogDir(), mapredUser, mapredGroup, Constants.PERMISSION_755, true); + LinuxFileUtils.createDirectories( + mapreduce2Params.getMapredPidDir(), mapredUser, mapredGroup, Constants.PERMISSION_755, true); + + // mapreduce.conf + log.info("Generating [{}/mapreduce.conf] file", BaseParams.LIMITS_CONF_DIR); + LinuxFileUtils.toFileByTemplate( + mapreduce2Params.mapredLimits(), + MessageFormat.format("{0}/mapreduce.conf", BaseParams.LIMITS_CONF_DIR), + Constants.ROOT_USER, + Constants.ROOT_USER, + Constants.PERMISSION_644, + mapreduce2Params.getGlobalParamsMap()); + + // mapred-env.sh + log.info("Generating [{}/mapred-env.sh] file", confDir); + LinuxFileUtils.toFileByTemplate( + mapreduce2Params.mapredEnv().get("content").toString(), + MessageFormat.format("{0}/mapred-env.sh", confDir), + mapredUser, + mapredGroup, + Constants.PERMISSION_644, + mapreduce2Params.getGlobalParamsMap()); + + // mapred-site.xml + log.info("Generating [{}/mapred-site.xml] file", confDir); + LinuxFileUtils.toFile( + ConfigType.XML, + MessageFormat.format("{0}/mapred-site.xml", confDir), + mapredUser, + mapredGroup, + Constants.PERMISSION_644, + mapreduce2Params.mapredSite(), + mapreduce2Params.getGlobalParamsMap()); + + HdfsUtil.createDirectory(hdfsUser, "/apps"); + HdfsUtil.createDirectory(hdfsUser, "/app-logs"); + HdfsUtil.createDirectory(mapredUser, "/apps/mapred"); + HdfsUtil.createDirectory(mapredUser, "/apps/mapred/staging"); + HdfsUtil.createDirectory(mapredUser, "/apps/mapred/history"); + HdfsUtil.createDirectory(mapredUser, "/apps/mapred/history/tmp"); + HdfsUtil.createDirectory(mapredUser, "/apps/mapred/history/done"); + + log.info("Successfully configured MapReduce2"); + return ShellResult.success("MapReduce2 Configure success!"); + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/Mapreduce2Params.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/Mapreduce2Params.java new file mode 100644 index 00000000..f690b09f --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/Mapreduce2Params.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.mapred; + +import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; +import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; +import org.apache.bigtop.manager.stack.core.param.BaseParams; +import org.apache.bigtop.manager.stack.core.utils.LocalSettings; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.text.MessageFormat; +import java.util.Map; + +@Getter +@Slf4j +public class Mapreduce2Params extends BaseParams { + + private String mapredEnvContent; + private String mapredLogDir; + private String mapredPidDir; + private String historyServerPidFile; + + public Mapreduce2Params(CommandPayload commandPayload) { + super(commandPayload); + globalParamsMap.put("mapred_user", user()); + globalParamsMap.put("mapred_group", group()); + globalParamsMap.put("java_home", "/usr/local/java"); + globalParamsMap.put("hadoop_home", serviceHome()); + globalParamsMap.put("hadoop_conf_dir", confDir()); + globalParamsMap.put("hadoop_hdfs_home", hdfsHome()); + globalParamsMap.put("hadoop_yarn_home", yarnHome()); + globalParamsMap.put("hadoop_mapred_home", mapredHome()); + globalParamsMap.put("hadoop_libexec_dir", serviceHome() + "/libexec"); + } + + public String mapredLimits() { + Map yarnConf = LocalSettings.configurations(serviceName(), "mapreduce.conf"); + return (String) yarnConf.get("content"); + } + + @GlobalParams + public Map mapredSite() { + return LocalSettings.configurations(serviceName(), "mapred-site"); + } + + @GlobalParams + public Map mapredEnv() { + Map mapredEnv = LocalSettings.configurations(serviceName(), "mapred-env"); + mapredEnvContent = (String) mapredEnv.get("content"); + mapredLogDir = (String) mapredEnv.get("mapred_log_dir_prefix"); + mapredPidDir = (String) mapredEnv.get("mapred_pid_dir_prefix"); + historyServerPidFile = MessageFormat.format("{0}/{1}/hadoop-{1}-historyserver.pid", mapredPidDir, user()); + return mapredEnv; + } + + @Override + public String confDir() { + return "/etc/hadoop/conf"; + } + + @Override + public String serviceHome() { + return stackLibDir() + "/hadoop"; + } + + public String hdfsHome() { + return stackLibDir() + "/hadoop-hdfs"; + } + + public String yarnHome() { + return stackLibDir() + "/hadoop-yarn"; + } + + public String mapredHome() { + return stackLibDir() + "/hadoop-mapreduce"; + } + + public String mapredExec() { + return stackBinDir() + "/mapred"; + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java index aaf4ac93..739bd304 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java @@ -47,7 +47,7 @@ public static ShellResult config(Params params) { String tezGroup = tezParams.group(); // tez-site - log.info("Generating {}/tez-site.xml file", confDir); + log.info("Generating [{}/tez-site.xml] file", confDir); LinuxFileUtils.toFile( ConfigType.XML, MessageFormat.format("{0}/tez-site.xml", confDir), @@ -58,7 +58,7 @@ public static ShellResult config(Params params) { tezParams.getGlobalParamsMap()); // tez-env - log.info("Generating {}/tez-env.sh file", confDir); + log.info("Generating [{}/tez-env.sh] file", confDir); LinuxFileUtils.toFileByTemplate( tezParams.getTezEnvContent(), MessageFormat.format("{0}/tez-env.sh", confDir), diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java index 980b0b01..ccce8203 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java @@ -78,7 +78,7 @@ public static ShellResult config(Params params, String componentName) { yarnParams.getYarnPidDir(), yarnUser, yarnGroup, Constants.PERMISSION_755, true); LinuxFileUtils.createDirectories(yarnParams.getTmpDir(), yarnUser, yarnGroup, Constants.PERMISSION_755, true); - // hdfs.limits + // yarn.conf LinuxFileUtils.toFileByTemplate( yarnParams.yarnLimits(), MessageFormat.format("{0}/yarn.conf", BaseParams.LIMITS_CONF_DIR), @@ -87,7 +87,7 @@ public static ShellResult config(Params params, String componentName) { Constants.PERMISSION_644, yarnParams.getGlobalParamsMap()); - // hadoop-env.sh + // yarn-env.sh LinuxFileUtils.toFileByTemplate( yarnEnv.get("content").toString(), MessageFormat.format("{0}/yarn-env.sh", confDir), @@ -96,7 +96,7 @@ public static ShellResult config(Params params, String componentName) { Constants.PERMISSION_644, yarnParams.getGlobalParamsMap()); - // hdfs-site.xml + // yarn-site.xml LinuxFileUtils.toFile( ConfigType.XML, MessageFormat.format("{0}/yarn-site.xml", confDir),