From 3d034624618823d2ff1e81c52ff91b011e290d9d Mon Sep 17 00:00:00 2001 From: Zhiguo Wu Date: Mon, 9 Dec 2024 15:59:57 +0800 Subject: [PATCH 1/2] Install Tez with tarball --- .../bigtop/3.3.0/services/flink/metainfo.xml | 6 ++- .../bigtop/3.3.0/services/hadoop/metainfo.xml | 2 +- .../bigtop/3.3.0/services/kafka/metainfo.xml | 4 +- .../bigtop/3.3.0/services/solr/metainfo.xml | 4 +- .../bigtop/3.3.0/services/tez/metainfo.xml | 6 ++- .../3.3.0/services/zookeeper/metainfo.xml | 2 +- .../bigtop/v3_3_0/tez/TezClientScript.java | 30 ++++++++++- .../stack/bigtop/v3_3_0/tez/TezParams.java | 15 +----- .../stack/bigtop/v3_3_0/tez/TezSetup.java | 18 +++---- .../stack/core/tarball/TarballDownloader.java | 53 ++++++++++++++++++- .../stack/core/tarball/TarballExtractor.java | 2 +- .../stack/core/utils/TarballUtils.java | 46 +--------------- 12 files changed, 108 insertions(+), 80 deletions(-) diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/metainfo.xml index dd989a05..6fc6bbd4 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/metainfo.xml @@ -52,17 +52,19 @@ x86_64 + aarch64 - flink_3_3_0 + flink-1.16.2-1.tgz + SHA-256:8c6da7cbab14b9cf0ce39c8c873b29f16c83df0b95b7af5cf3061a468efac228 - yarn + hadoop \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml index ce0615be..0ced8143 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml @@ -112,7 +112,7 @@ hadoop-3.3.6-1.tar.gz - SHA-256:2a9312eac69a542c6266f1689d29ded243dde2d61add333a9762829008f97442 + SHA-256:0da6364126c6ea6dd0f5b545afec33957ee801d6db6c0874edb0aa5be1e89995 diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml index 2c113d53..75ed8e85 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml @@ -43,10 +43,12 @@ x86_64 + aarch64 - kafka_3_3_0 + kafka-2.8.2-1.tgz + SHA-256:30a2d69ef081813624273d8a406c9b803c5868df998484e27207f85ea217870f diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/metainfo.xml index 95ee24a0..32c5465f 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/metainfo.xml @@ -47,10 +47,12 @@ x86_64 + aarch64 - solr_3_3_0 + solr-8.11.2-2.tgz + SHA-256:fba6618e4c3a3133a4c4d57718e52cc797d10374f056825329a9e76b93de9f48 diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml index 3ca39261..509db831 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml @@ -39,17 +39,19 @@ x86_64 + aarch64 - tez_3_3_0 + tez-0.10.2-1.tar.gz + SHA-256:ebc2c195780fc76fb9b12e8987e48c301ca0b1916dca515f5c7a1122d74f397e - hdfs + hadoop diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml index 7b35ebff..7748ff51 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml @@ -54,7 +54,7 @@ zookeeper-3.7.2-1.tar.gz - SHA-256:380f15d55c0282e33fdc7c2ec551bc5586f0ac126c243cd84347ccb775b846f3 + SHA-256:83e07f914eb3477c77245f4dc44031b82ea6ef1be3691687f469ddc4b8c720bb diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java index e65478b7..bcbea7d9 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java @@ -18,21 +18,49 @@ */ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez; +import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo; +import org.apache.bigtop.manager.common.message.entity.pojo.RepoInfo; import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; +import org.apache.bigtop.manager.stack.core.tarball.TarballDownloader; import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; +import java.io.File; +import java.util.List; + @Slf4j @AutoService(Script.class) public class TezClientScript extends AbstractClientScript { + /** + * Tez tarball file doesn't need to be extracted, so here we override {@link #add(Params)} method + * and do nothing after download the tarball file. + * + * @param params the parameters required for installation + * @return ShellResult + */ + @Override + public ShellResult add(Params params) { + RepoInfo repo = params.repo(); + List packages = params.packages(); + + String repoUrl = repo.getBaseUrl(); + String stackHome = params.stackHome(); + for (PackageInfo packageInfo : packages) { + String remoteUrl = repoUrl + File.separator + packageInfo.getName(); + TarballDownloader.download(remoteUrl, stackHome, packageInfo); + } + + return ShellResult.success(); + } + @Override public ShellResult configure(Params params) { - return TezSetup.config(params); + return TezSetup.configure(params); } @Override diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java index 25995737..3d2c8784 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java @@ -77,27 +77,14 @@ public Map tezEnv() { return tezEnv; } - @Override - public String confDir() { - return "/etc/tez/conf"; - } - public String hadoopConfDir() { - return "/etc/hadoop/conf"; + return hadoopHome() + "/etc/hadoop"; } public String hadoopHome() { return stackHome() + "/hadoop"; } - public String hdfsHome() { - return stackHome() + "/hadoop-hdfs"; - } - - public String yarnHome() { - return stackHome() + "/hadoop-yarn"; - } - @Override public String getServiceName() { return "tez"; diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java index cd995251..67a69906 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java @@ -20,7 +20,6 @@ import org.apache.bigtop.manager.common.constants.Constants; import org.apache.bigtop.manager.common.shell.ShellResult; -import org.apache.bigtop.manager.stack.bigtop.utils.HdfsUtil; import org.apache.bigtop.manager.stack.core.enums.ConfigType; import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; @@ -38,16 +37,17 @@ @NoArgsConstructor(access = AccessLevel.PRIVATE) public class TezSetup { - public static ShellResult config(Params params) { + public static ShellResult configure(Params params) { + log.info("Configuring Tez"); TezParams tezParams = (TezParams) params; String confDir = tezParams.confDir(); - String hdfsUser = LocalSettings.users().get("hdfs"); + String hadoopUser = LocalSettings.users().get("hadoop"); String tezUser = tezParams.user(); String tezGroup = tezParams.group(); - // tez-site - log.info("Generating [{}/tez-site.xml] file", confDir); + LinuxFileUtils.createDirectories(confDir, tezUser, tezGroup, Constants.PERMISSION_755, true); + LinuxFileUtils.toFile( ConfigType.XML, MessageFormat.format("{0}/tez-site.xml", confDir), @@ -57,8 +57,6 @@ public static ShellResult config(Params params) { tezParams.tezSite(), tezParams.getGlobalParamsMap()); - // tez-env - log.info("Generating [{}/tez-env.sh] file", confDir); LinuxFileUtils.toFileByTemplate( tezParams.getTezEnvContent(), MessageFormat.format("{0}/tez-env.sh", confDir), @@ -67,10 +65,10 @@ public static ShellResult config(Params params) { PERMISSION_755, tezParams.getGlobalParamsMap()); - HdfsUtil.createDirectory(hdfsUser, "/apps"); - HdfsUtil.uploadFile(tezUser, tezParams.serviceHome() + "/lib/tez.tar.gz", "/apps/tez"); + // HdfsUtil.createDirectory(hadoopUser, "/apps"); + // HdfsUtil.uploadFile(tezUser, tezParams.serviceHome() + "/lib/tez.tar.gz", "/apps/tez"); log.info("Successfully configured Tez"); - return ShellResult.success("Tez Configure success!"); + return ShellResult.success(); } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java index 18827917..b0ca3d53 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java @@ -18,6 +18,9 @@ */ package org.apache.bigtop.manager.stack.core.tarball; +import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo; +import org.apache.bigtop.manager.stack.core.exception.StackException; + import lombok.extern.slf4j.Slf4j; import java.io.File; @@ -30,7 +33,55 @@ public class TarballDownloader { @SuppressWarnings("ResultOfMethodCallIgnored") - public static Boolean downloadFile(String fileUrl, String saveDir) { + public static void download(String remoteUrl, String saveDir, PackageInfo packageInfo) { + File localFile = new File(saveDir + File.separator + packageInfo.getName()); + String algorithm = packageInfo.getChecksum().split(":")[0]; + String checksum = packageInfo.getChecksum().split(":")[1]; + + if (localFile.exists()) { + log.info("File [{}] exists, validating checksum", localFile.getAbsolutePath()); + } else { + log.info("Downloading [{}] to [{}]", remoteUrl, saveDir); + download(remoteUrl, saveDir); + } + + boolean validateChecksum = ChecksumValidator.validateChecksum(algorithm, checksum, localFile); + if (!validateChecksum) { + log.warn("Invalid checksum for [{}], re-downloading...", localFile.getAbsolutePath()); + localFile.delete(); + download(remoteUrl, saveDir); + } + + validateChecksum = ChecksumValidator.validateChecksum(algorithm, checksum, localFile); + if (!validateChecksum) { + log.error("Invalid checksum for [{}], exiting...", localFile.getAbsolutePath()); + throw new StackException("Invalid checksum for " + localFile.getAbsolutePath()); + } + + log.info("Checksum validate successfully for [{}]", localFile.getAbsolutePath()); + } + + private static void download(String remoteUrl, String saveDir) { + int i = 1; + while (true) { + Boolean downloaded = downloadFile(remoteUrl, saveDir); + if (downloaded) { + break; + } else { + if (i == 3) { + log.error("Failed to download [{}], exiting...", remoteUrl); + throw new StackException("Failed to download " + remoteUrl); + } else { + log.error("Failed to download [{}], retrying...: {}", remoteUrl, i); + } + } + + i++; + } + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + private static Boolean downloadFile(String fileUrl, String saveDir) { HttpURLConnection httpConn = null; try { URL url = new URL(fileUrl); diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java index ef3c9f7a..f0d99c50 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java @@ -126,7 +126,7 @@ private static void extractTarGz(File tarball, Function Date: Mon, 9 Dec 2024 16:37:49 +0800 Subject: [PATCH 2/2] copy tez.tar.gz to tez lib --- .../stack/bigtop/v3_3_0/tez/TezSetup.java | 14 +++++++++++ .../core/utils/linux/LinuxFileUtils.java | 25 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java index 67a69906..b68c77ad 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java @@ -19,6 +19,7 @@ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez; import org.apache.bigtop.manager.common.constants.Constants; +import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo; import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.stack.core.enums.ConfigType; import org.apache.bigtop.manager.stack.core.spi.param.Params; @@ -30,6 +31,7 @@ import lombok.extern.slf4j.Slf4j; import java.text.MessageFormat; +import java.util.List; import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_755; @@ -42,11 +44,23 @@ public static ShellResult configure(Params params) { TezParams tezParams = (TezParams) params; String confDir = tezParams.confDir(); + String serviceHome = tezParams.serviceHome(); + String libDir = serviceHome + "/lib"; String hadoopUser = LocalSettings.users().get("hadoop"); String tezUser = tezParams.user(); String tezGroup = tezParams.group(); + LinuxFileUtils.createDirectories(serviceHome, tezUser, tezGroup, Constants.PERMISSION_755, true); LinuxFileUtils.createDirectories(confDir, tezUser, tezGroup, Constants.PERMISSION_755, true); + LinuxFileUtils.createDirectories(libDir, tezUser, tezGroup, Constants.PERMISSION_755, true); + + List packages = params.packages(); + for (PackageInfo packageInfo : packages) { + String source = tezParams.stackHome() + "/" + packageInfo.getName(); + String dest = libDir + "/tez.tar.gz"; + LinuxFileUtils.copyFile(source, dest); + LinuxFileUtils.updateOwner(dest, tezUser, tezGroup, false); + } LinuxFileUtils.toFile( ConfigType.XML, diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxFileUtils.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxFileUtils.java index 623862f2..50858db8 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxFileUtils.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxFileUtils.java @@ -208,6 +208,31 @@ public static void removeDirectories(String dirPath) { } } + public static void copyFile(String source, String dest) { + if (StringUtils.isBlank(source) || StringUtils.isBlank(dest)) { + log.error("source and dest must not be empty"); + return; + } + + List builderParameters = new ArrayList<>(); + builderParameters.add("cp"); + if (Files.exists(Path.of(source)) && Files.isDirectory(Paths.get(source))) { + builderParameters.add("-r"); + } + + builderParameters.add(source); + builderParameters.add(dest); + + try { + ShellResult shellResult = sudoExecCmd(builderParameters); + if (shellResult.getExitCode() != MessageConstants.SUCCESS_CODE) { + throw new StackException(shellResult.getErrMsg()); + } + } catch (IOException e) { + throw new StackException(e); + } + } + public static void moveFile(String source, String dest) { if (StringUtils.isBlank(source) || StringUtils.isBlank(dest)) { log.error("source and dest must not be empty");