BIGTOP-4307: Add task for cluster jdk setup

bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/executor/SetupJdkCommandExecutor.java

@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.agent.executor;
+
+import org.apache.bigtop.manager.common.constants.Constants;
+import org.apache.bigtop.manager.common.message.entity.pojo.ClusterInfo;
+import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo;
+import org.apache.bigtop.manager.common.message.entity.pojo.RepoInfo;
+import org.apache.bigtop.manager.common.utils.os.OSDetection;
+import org.apache.bigtop.manager.grpc.generated.CommandType;
+import org.apache.bigtop.manager.stack.core.utils.LocalSettings;
+import org.apache.bigtop.manager.stack.core.utils.TarballUtils;
+import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils;
+
+import org.springframework.beans.factory.config.ConfigurableBeanFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.text.MessageFormat;
+
+@Slf4j
+@Component
+@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
+public class SetupJdkCommandExecutor extends AbstractCommandExecutor {
+
+    @Override
+    public CommandType getCommandType() {
+        return CommandType.SETUP_JDK;
+    }
+
+    /**
+     * TODO JDK info currently is hardcoded, need to improve and make it configurable.
+     */
+    @Override
+    public void doExecute() {
+        log.info("Setting up cluster jdk...");
+        String arch = OSDetection.getArch();
+        String pkgName = getPkgName(arch);
+        String checksum = getChecksum(arch);
+        PackageInfo packageInfo = new PackageInfo(pkgName, checksum);
+        ClusterInfo clusterInfo = LocalSettings.cluster();
+        RepoInfo repoInfo = LocalSettings.repos().stream()
+                .filter(r -> arch.equals(r.getArch()) && r.getType() == 2)
+                .findFirst()
+                .orElseThrow(() -> new RuntimeException(
+                        "Cannot find repo for os: [" + OSDetection.getOS() + "] and arch: [" + arch + "]"));
+        String toolsHome = clusterInfo.getRootDir() + "/tools";
+        String user = System.getProperty("user.name");
+        LinuxFileUtils.createDirectories(toolsHome, user, user, Constants.PERMISSION_755, true);
+
+        String jdkHome = toolsHome + "/jdk";
+        TarballUtils.installPackage(repoInfo.getBaseUrl(), toolsHome, jdkHome, packageInfo, 1);
+        LinuxFileUtils.createDirectories(jdkHome, user, user, Constants.PERMISSION_755, true);
+    }
+
+    private String getPkgName(String arch) {
+        String replacedArch = arch.equals("x86_64") ? "x64" : arch;
+        replacedArch = replacedArch.equals("arm64") ? "aarch64" : arch;
+        return MessageFormat.format("jdk-8u431-linux-{0}.tar.gz", replacedArch);
+    }
+
+    private String getChecksum(String arch) {
+        return switch (arch) {
+            case "x64", "x86_64" -> "SHA-256:b396978a716b7d23ccccabfe5c47c3b75d2434d7f8f7af690bc648172382720d";
+            case "arm64", "aarch64" -> "SHA-256:e68d3e31ffcf7f05a4de65d04974843073bdff238bb6524adb272de9e616be7c";
+            default -> throw new RuntimeException("Unknown arch for jdk: " + arch);
+        };
+    }
+}

bigtop-manager-grpc/src/main/resources/proto/command.proto

@@ -30,6 +30,7 @@ enum CommandType {
   COMPONENT = 0;
   HOST_CHECK = 1;
   UPDATE_CACHE_FILES = 2;
+  SETUP_JDK = 3;
 }
 
 message CommandRequest {

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ClusterAddJob.java

@@ -25,6 +25,7 @@
 import org.apache.bigtop.manager.dao.repository.HostDao;
 import org.apache.bigtop.manager.server.command.stage.CacheFileUpdateStage;
 import org.apache.bigtop.manager.server.command.stage.HostCheckStage;
+import org.apache.bigtop.manager.server.command.stage.SetupJdkStage;
 import org.apache.bigtop.manager.server.command.stage.Stage;
 import org.apache.bigtop.manager.server.command.stage.StageContext;
 import org.apache.bigtop.manager.server.command.task.Task;
@@ -57,6 +58,7 @@ protected void createStages() {
         StageContext stageContext = StageContext.fromCommandDTO(jobContext.getCommandDTO());
         stages.add(new HostCheckStage(stageContext));
         stages.add(new CacheFileUpdateStage(stageContext));
+        stages.add(new SetupJdkStage(stageContext));
     }
 
     @Override

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/SetupJdkStage.java

@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.server.command.stage;
+
+import org.apache.bigtop.manager.server.command.task.SetupJdkTask;
+import org.apache.bigtop.manager.server.command.task.Task;
+import org.apache.bigtop.manager.server.command.task.TaskContext;
+import org.apache.bigtop.manager.server.model.dto.HostDTO;
+
+public class SetupJdkStage extends AbstractStage {
+
+    public SetupJdkStage(StageContext stageContext) {
+        super(stageContext);
+    }
+
+    @Override
+    protected void injectBeans() {
+        super.injectBeans();
+    }
+
+    @Override
+    protected void beforeCreateTasks() {}
+
+    @Override
+    protected Task createTask(HostDTO hostDTO) {
+        TaskContext taskContext = new TaskContext();
+        taskContext.setHostDTO(hostDTO);
+        taskContext.setClusterId(stageContext.getClusterId());
+        taskContext.setClusterName(stageContext.getClusterName());
+        taskContext.setServiceName("cluster");
+        taskContext.setServiceUser("root");
+        taskContext.setComponentName("agent");
+        taskContext.setComponentDisplayName("Agent");
+
+        return new SetupJdkTask(taskContext);
+    }
+
+    @Override
+    public String getName() {
+        return "Setup JDK";
+    }
+}

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/SetupJdkTask.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.server.command.task;
+
+import org.apache.bigtop.manager.common.enums.Command;
+import org.apache.bigtop.manager.grpc.generated.CommandRequest;
+import org.apache.bigtop.manager.grpc.generated.CommandType;
+
+public class SetupJdkTask extends AbstractTask {
+
+    public SetupJdkTask(TaskContext taskContext) {
+        super(taskContext);
+    }
+
+    @Override
+    protected Command getCommand() {
+        return Command.CUSTOM;
+    }
+
+    @Override
+    protected String getCustomCommand() {
+        return "setup_jdk";
+    }
+
+    @Override
+    protected CommandRequest getCommandRequest() {
+        CommandRequest.Builder builder = CommandRequest.newBuilder();
+        builder.setType(CommandType.SETUP_JDK);
+        return builder.build();
+    }
+
+    @Override
+    public String getName() {
+        return "Setup jdk for " + taskContext.getHostDTO().getHostname();
+    }
+}

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/mapred-env.xml

@@ -39,7 +39,6 @@ export HADOOP_HOME=${hadoop_home}
 # Could be enabled from deployment option if necessary
 export MAPRED_HISTORYSERVER_OPTS="$MAPRED_HISTORYSERVER_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
 export HADOOP_LIBEXEC_DIR=${hadoop_libexec_dir}
-export HADOOP_OPTS="$HADOOP_OPTS --add-opens java.base/java.lang=ALL-UNNAMED"
 export JAVA_HOME=${java_home}
 ]]>
         </value>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/yarn-env.xml

@@ -166,7 +166,6 @@ HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
 fi
 HADOOP_OPTS="$HADOOP_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
 </#noparse>
-HADOOP_OPTS="$HADOOP_OPTS --add-opens java.base/java.lang=ALL-UNNAMED"
 ]]>
         </value>
         <attrs>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hbase/metainfo.xml

@@ -77,7 +77,6 @@
         </package-specifics>
 
         <required-services>
-            <service>zookeeper</service>
             <service>hadoop</service>
         </required-services>
 

bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkParams.java

@@ -19,7 +19,6 @@
 package org.apache.bigtop.manager.stack.bigtop.v3_3_0.flink;
 
 import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload;
-import org.apache.bigtop.manager.common.utils.Environments;
 import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams;
 import org.apache.bigtop.manager.stack.core.annotations.GlobalParams;
 import org.apache.bigtop.manager.stack.core.spi.param.Params;
@@ -57,7 +56,7 @@ public FlinkParams(CommandPayload commandPayload) {
         super(commandPayload);
         globalParamsMap.put("flink_user", user());
         globalParamsMap.put("flink_group", group());
-        globalParamsMap.put("java_home", Environments.getJavaHome());
+        globalParamsMap.put("java_home", javaHome());
         globalParamsMap.put("hadoop_home", hadoopHome());
         globalParamsMap.put("hadoop_conf_dir", hadoopConfDir());
 

bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HadoopParams.java

@@ -19,7 +19,6 @@
 package org.apache.bigtop.manager.stack.bigtop.v3_3_0.hadoop;
 
 import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload;
-import org.apache.bigtop.manager.common.utils.Environments;
 import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams;
 import org.apache.bigtop.manager.stack.core.annotations.GlobalParams;
 import org.apache.bigtop.manager.stack.core.spi.param.Params;
@@ -72,7 +71,7 @@ public HadoopParams(CommandPayload commandPayload) {
         globalParamsMap.put("hdfs_user", user());
         globalParamsMap.put("hdfs_group", group());
         globalParamsMap.put("datanode_hosts", LocalSettings.hosts("datanode"));
-        globalParamsMap.put("java_home", Environments.getJavaHome());
+        globalParamsMap.put("java_home", javaHome());
         globalParamsMap.put("hadoop_home", serviceHome());
         globalParamsMap.put("hadoop_conf_dir", confDir());
         globalParamsMap.put("hadoop_libexec_dir", serviceHome() + "/libexec");

bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hbase/HBaseParams.java

@@ -19,7 +19,6 @@
 package org.apache.bigtop.manager.stack.bigtop.v3_3_0.hbase;
 
 import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload;
-import org.apache.bigtop.manager.common.utils.Environments;
 import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams;
 import org.apache.bigtop.manager.stack.core.annotations.GlobalParams;
 import org.apache.bigtop.manager.stack.core.spi.param.Params;
@@ -51,7 +50,7 @@ public class HBaseParams extends BigtopParams {
 
     public HBaseParams(CommandPayload commandPayload) {
         super(commandPayload);
-        globalParamsMap.put("java_home", Environments.getJavaHome());
+        globalParamsMap.put("java_home", javaHome());
         globalParamsMap.put("hbase_home", serviceHome());
         globalParamsMap.put("hbase_conf_dir", confDir());
         globalParamsMap.put("security_enabled", false);

bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaParams.java

@@ -19,7 +19,6 @@
 package org.apache.bigtop.manager.stack.bigtop.v3_3_0.kafka;
 
 import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload;
-import org.apache.bigtop.manager.common.utils.Environments;
 import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams;
 import org.apache.bigtop.manager.stack.core.annotations.GlobalParams;
 import org.apache.bigtop.manager.stack.core.spi.param.Params;
@@ -47,7 +46,7 @@ public KafkaParams(CommandPayload commandPayload) {
         super(commandPayload);
         globalParamsMap.put("kafka_user", user());
         globalParamsMap.put("kafka_group", group());
-        globalParamsMap.put("java_home", Environments.getJavaHome());
+        globalParamsMap.put("java_home", javaHome());
         globalParamsMap.put("kafka_conf_dir", confDir());
         globalParamsMap.put("security_enabled", false);
     }

bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrParams.java

@@ -19,7 +19,6 @@
 package org.apache.bigtop.manager.stack.bigtop.v3_3_0.solr;
 
 import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload;
-import org.apache.bigtop.manager.common.utils.Environments;
 import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams;
 import org.apache.bigtop.manager.stack.core.annotations.GlobalParams;
 import org.apache.bigtop.manager.stack.core.spi.param.Params;
@@ -51,7 +50,7 @@ public class SolrParams extends BigtopParams {
 
     public SolrParams(CommandPayload commandPayload) {
         super(commandPayload);
-        globalParamsMap.put("java_home", Environments.getJavaHome());
+        globalParamsMap.put("java_home", javaHome());
         globalParamsMap.put("solr_home", serviceHome());
         globalParamsMap.put("security_enabled", false);
         globalParamsMap.put("solr_pid_file", solrPidFile);

bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java

@@ -19,7 +19,6 @@
 package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez;
 
 import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload;
-import org.apache.bigtop.manager.common.utils.Environments;
 import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams;
 import org.apache.bigtop.manager.stack.core.annotations.GlobalParams;
 import org.apache.bigtop.manager.stack.core.spi.param.Params;
@@ -48,7 +47,7 @@ public TezParams(CommandPayload commandPayload) {
         super(commandPayload);
         globalParamsMap.put("tez_user", user());
         globalParamsMap.put("tez_group", group());
-        globalParamsMap.put("java_home", Environments.getJavaHome());
+        globalParamsMap.put("java_home", javaHome());
         globalParamsMap.put("hadoop_home", hadoopHome());
         globalParamsMap.put("hadoop_conf_dir", hadoopConfDir());
         globalParamsMap.put("tez_home", serviceHome());

bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperParams.java

@@ -19,7 +19,6 @@
 package org.apache.bigtop.manager.stack.bigtop.v3_3_0.zookeeper;
 
 import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload;
-import org.apache.bigtop.manager.common.utils.Environments;
 import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams;
 import org.apache.bigtop.manager.stack.core.annotations.GlobalParams;
 import org.apache.bigtop.manager.stack.core.spi.param.Params;
@@ -44,7 +43,7 @@ public class ZookeeperParams extends BigtopParams {
 
     public ZookeeperParams(CommandPayload commandPayload) {
         super(commandPayload);
-        globalParamsMap.put("java_home", Environments.getJavaHome());
+        globalParamsMap.put("java_home", javaHome());
         globalParamsMap.put("zookeeper_home", serviceHome());
         globalParamsMap.put("security_enabled", false);
         globalParamsMap.put("zookeeper_pid_file", zookeeperPidFile);

bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParams.java

@@ -116,6 +116,12 @@ public List<PackageInfo> packages() {
         return packageInfoList;
     }
 
+    @Override
+    public String javaHome() {
+        String root = LocalSettings.cluster().getRootDir();
+        return MessageFormat.format("{0}/tools/jdk", root);
+    }
+
     @Override
     public String stackHome() {
         String root = LocalSettings.cluster().getRootDir();

bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/Params.java

@@ -36,10 +36,12 @@ public interface Params extends PrioritySPI {
 
     List<PackageInfo> packages();
 
-    String serviceHome();
+    String javaHome();
 
     String stackHome();
 
+    String serviceHome();
+
     String getServiceName();
 
     @Override