Skip to content

Commit

Permalink
BIGTOP-4131: Add Tez component on Bigtop-3.3.0 stack
Browse files Browse the repository at this point in the history
  • Loading branch information
kevinw66 committed Aug 7, 2024
1 parent e8b6f6a commit 5e1d6d3
Show file tree
Hide file tree
Showing 36 changed files with 1,115 additions and 101 deletions.
29 changes: 29 additions & 0 deletions bigtop-manager-agent/src/main/resources/bin/start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,41 @@ BIGTOP_MANAGER_HOME=${BIGTOP_MANAGER_HOME:-$(cd $BIN_DIR/..; pwd)}

source "${BIGTOP_MANAGER_HOME}/bin/env.sh"

usage() {
echo "usage: $PROG [--debug]"
echo " --debug - enable debug mode"
echo " -h, --help"
exit 1
}

DEBUG="false"
DOCKER="false"

while [ $# -gt 0 ]; do
case "$1" in
--debug)
echo "enable debug mode."
DEBUG="true"
shift;;
-h|--help)
usage
shift;;
*)
echo "Unknown argument: '$1'" 1>&2
usage;;
esac
done

JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms4g -Xmx4g -Xmn2g -XX:+IgnoreUnrecognizedVMOptions -XX:+PrintGCDateStamps -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}

if [[ "$DOCKER" == "true" ]]; then
JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport"
fi

if [[ "$DEBUG" == "true" ]]; then
JAVA_OPTS="${JAVA_OPTS} -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006"
fi

cd $BIGTOP_MANAGER_HOME

$JAVA_HOME/bin/java $JAVA_OPTS \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,15 @@
import org.apache.bigtop.manager.dao.repository.HostComponentRepository;
import org.apache.bigtop.manager.server.command.stage.CacheFileUpdateStage;
import org.apache.bigtop.manager.server.command.stage.ComponentCheckStage;
import org.apache.bigtop.manager.server.command.stage.ComponentConfigureStage;
import org.apache.bigtop.manager.server.command.stage.ComponentInstallStage;
import org.apache.bigtop.manager.server.command.stage.ComponentStartStage;
import org.apache.bigtop.manager.server.command.stage.ComponentStopStage;
import org.apache.bigtop.manager.server.command.stage.StageContext;
import org.apache.bigtop.manager.server.exception.ServerException;
import org.apache.bigtop.manager.server.holder.SpringContextHolder;
import org.apache.bigtop.manager.server.model.dto.ComponentDTO;
import org.apache.bigtop.manager.server.model.dto.ComponentHostDTO;
import org.apache.bigtop.manager.server.model.dto.ServiceDTO;
import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO;
import org.apache.bigtop.manager.server.stack.dag.ComponentCommandWrapper;
Expand Down Expand Up @@ -178,6 +180,23 @@ protected void createInstallStages() {
}
}

protected void createConfigureStages() {
for (ServiceCommandDTO serviceCommand : jobContext.getCommandDTO().getServiceCommands()) {
if (serviceCommand.getInstalled()) {
continue;
}

for (ComponentHostDTO componentHost : serviceCommand.getComponentHosts()) {
String serviceName = serviceCommand.getServiceName();
String componentName = componentHost.getComponentName();
List<String> hostnames = componentHost.getHostnames();

StageContext stageContext = createStageContext(serviceName, componentName, hostnames);
stages.add(new ComponentConfigureStage(stageContext));
}
}
}

protected void createStartStages() {
List<String> todoList = getTodoListForCommand(Command.START);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,15 @@ public ServiceConfigureJob(JobContext jobContext) {

@Override
protected void createStages() {
// Update cache files
super.createCacheStage();

// Configure services
super.createConfigureStages();

// Restart services
super.createStopStages();
super.createStartStages();
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,9 @@ protected void createStages() {
// Update cache files after installed
super.createCacheStage();

// Configure services
super.createConfigureStages();

// Start all master components
super.createStartStages();

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.bigtop.manager.server.command.stage;

import org.apache.bigtop.manager.server.command.task.ComponentConfigureTask;
import org.apache.bigtop.manager.server.command.task.Task;

public class ComponentConfigureStage extends AbstractComponentStage {

public ComponentConfigureStage(StageContext stageContext) {
super(stageContext);
}

@Override
protected Task createTask(String hostname) {
return new ComponentConfigureTask(createTaskContext(hostname));
}

@Override
public String getName() {
return "Configure " + stageContext.getComponentDTO().getDisplayName();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.bigtop.manager.server.command.task;

import org.apache.bigtop.manager.common.enums.Command;

public class ComponentConfigureTask extends AbstractComponentTask {

public ComponentConfigureTask(TaskContext taskContext) {
super(taskContext);
}

@Override
protected Command getCommand() {
return Command.CONFIGURE;
}

@Override
public String getName() {
return "Configure " + taskContext.getComponentDisplayName() + " on " + taskContext.getHostname();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,6 @@
@XmlAccessorType(XmlAccessType.FIELD)
public class ConfigurationXml {

@XmlElement(name = "schema-version")
private String schemaVersion;

@XmlElement(name = "property")
private List<PropertyModel> propertyModels;
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,12 @@

import jakarta.xml.bind.annotation.XmlAccessType;
import jakarta.xml.bind.annotation.XmlAccessorType;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlRootElement;

@Data
@XmlRootElement(name = "metainfo")
@XmlAccessorType(XmlAccessType.FIELD)
public class ServiceMetainfoXml {

@XmlElement(name = "schema-version")
private String schemaVersion;

private ServiceModel service;
}
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,6 @@
@XmlAccessorType(XmlAccessType.FIELD)
public class StackMetainfoXml {

@XmlElement(name = "schema-version")
private String schemaVersion;

@XmlElement(name = "stack")
private StackModel stack;
}
1 change: 0 additions & 1 deletion bigtop-manager-server/src/main/resources/bin/start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ if [[ "$DEBUG" == "true" ]]; then
JAVA_OPTS="${JAVA_OPTS} -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005"
fi


cd $BIGTOP_MANAGER_HOME

$JAVA_HOME/bin/java $JAVA_OPTS \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
-->

<metainfo>
<schema-version>2.0</schema-version>
<stack>
<stack-name>bigtop</stack-name>
<stack-version>3.3.0</stack-version>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
-->

<metainfo>
<schema-version>2.0</schema-version>
<service>
<name>hdfs</name>
<display-name>HDFS</display-name>
Expand Down Expand Up @@ -113,7 +112,7 @@
<package>hadoop_3_3_0</package>
<package>hadoop_3_3_0-client</package>
<package>hadoop_3_3_0-libhdfs</package>
</packages>
</packages>c
</os-specific>
</os-specifics>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
-->

<metainfo>
<schema-version>2.0</schema-version>
<service>
<name>kafka</name>
<display-name>Kafka</display-name>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,5 @@
],
"KAFKA_BROKER-RESTART": [
"ZOOKEEPER_SERVER-RESTART"
],
"ZOOKEEPER_SERVER-STOP": [
"KAFKA_BROKER-STOP"
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
<?xml version="1.0"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing,
~ software distributed under the License is distributed on an
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
~ KIND, either express or implied. See the License for the
~ specific language governing permissions and limitations
~ under the License.
-->

<configuration>
<!-- tez-env.sh -->
<property>
<name>content</name>
<display-name>tez-env template</display-name>
<description>This is the freemarker template for tez-env.sh file</description>
<value><![CDATA[
# Tez specific configuration
export TEZ_CONF_DIR=${tez_conf_dir!}
# Set HADOOP_HOME to point to a specific hadoop install directory
export HADOOP_HOME=${hadoop_home!}
# The java implementation to use.
export JAVA_HOME=${java_home!}
]]>
</value>
<attrs>
<type>longtext</type>
</attrs>
</property>
<property>
<name>enable_heap_dump</name>
<value>false</value>
<description>Enable or disable taking Heap Dump. (true/false)</description>
<display-name>Enable heap dump</display-name>
</property>
<property>
<name>heap_dump_location</name>
<value>/tmp</value>
<description>Location for heap dump file</description>
<display-name>Heap dump location</display-name>
</property>
</configuration>
Loading

0 comments on commit 5e1d6d3

Please sign in to comment.