diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/metainfo.xml index dd989a05e..6fc6bbd49 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/metainfo.xml @@ -52,17 +52,19 @@ x86_64 + aarch64 - flink_3_3_0 + flink-1.16.2-1.tgz + SHA-256:8c6da7cbab14b9cf0ce39c8c873b29f16c83df0b95b7af5cf3061a468efac228 - yarn + hadoop \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml index ce0615be0..0ced8143c 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml @@ -112,7 +112,7 @@ hadoop-3.3.6-1.tar.gz - SHA-256:2a9312eac69a542c6266f1689d29ded243dde2d61add333a9762829008f97442 + SHA-256:0da6364126c6ea6dd0f5b545afec33957ee801d6db6c0874edb0aa5be1e89995 diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml index 2c113d538..75ed8e854 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml @@ -43,10 +43,12 @@ x86_64 + aarch64 - kafka_3_3_0 + kafka-2.8.2-1.tgz + SHA-256:30a2d69ef081813624273d8a406c9b803c5868df998484e27207f85ea217870f diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/metainfo.xml index 95ee24a02..32c5465f4 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/metainfo.xml @@ -47,10 +47,12 @@ x86_64 + aarch64 - solr_3_3_0 + solr-8.11.2-2.tgz + SHA-256:fba6618e4c3a3133a4c4d57718e52cc797d10374f056825329a9e76b93de9f48 diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml index 3ca392610..509db8315 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml @@ -39,17 +39,19 @@ x86_64 + aarch64 - tez_3_3_0 + tez-0.10.2-1.tar.gz + SHA-256:ebc2c195780fc76fb9b12e8987e48c301ca0b1916dca515f5c7a1122d74f397e - hdfs + hadoop diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml index 7b35ebffb..7748ff51c 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml @@ -54,7 +54,7 @@ zookeeper-3.7.2-1.tar.gz - SHA-256:380f15d55c0282e33fdc7c2ec551bc5586f0ac126c243cd84347ccb775b846f3 + SHA-256:83e07f914eb3477c77245f4dc44031b82ea6ef1be3691687f469ddc4b8c720bb diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java index e65478b7f..bcbea7d92 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java @@ -18,21 +18,49 @@ */ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez; +import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo; +import org.apache.bigtop.manager.common.message.entity.pojo.RepoInfo; import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; +import org.apache.bigtop.manager.stack.core.tarball.TarballDownloader; import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; +import java.io.File; +import java.util.List; + @Slf4j @AutoService(Script.class) public class TezClientScript extends AbstractClientScript { + /** + * Tez tarball file doesn't need to be extracted, so here we override {@link #add(Params)} method + * and do nothing after download the tarball file. + * + * @param params the parameters required for installation + * @return ShellResult + */ + @Override + public ShellResult add(Params params) { + RepoInfo repo = params.repo(); + List packages = params.packages(); + + String repoUrl = repo.getBaseUrl(); + String stackHome = params.stackHome(); + for (PackageInfo packageInfo : packages) { + String remoteUrl = repoUrl + File.separator + packageInfo.getName(); + TarballDownloader.download(remoteUrl, stackHome, packageInfo); + } + + return ShellResult.success(); + } + @Override public ShellResult configure(Params params) { - return TezSetup.config(params); + return TezSetup.configure(params); } @Override diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java index 25995737c..3d2c87846 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java @@ -77,27 +77,14 @@ public Map tezEnv() { return tezEnv; } - @Override - public String confDir() { - return "/etc/tez/conf"; - } - public String hadoopConfDir() { - return "/etc/hadoop/conf"; + return hadoopHome() + "/etc/hadoop"; } public String hadoopHome() { return stackHome() + "/hadoop"; } - public String hdfsHome() { - return stackHome() + "/hadoop-hdfs"; - } - - public String yarnHome() { - return stackHome() + "/hadoop-yarn"; - } - @Override public String getServiceName() { return "tez"; diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java index cd995251e..b68c77ad6 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java @@ -19,8 +19,8 @@ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez; import org.apache.bigtop.manager.common.constants.Constants; +import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo; import org.apache.bigtop.manager.common.shell.ShellResult; -import org.apache.bigtop.manager.stack.bigtop.utils.HdfsUtil; import org.apache.bigtop.manager.stack.core.enums.ConfigType; import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; @@ -31,6 +31,7 @@ import lombok.extern.slf4j.Slf4j; import java.text.MessageFormat; +import java.util.List; import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_755; @@ -38,16 +39,29 @@ @NoArgsConstructor(access = AccessLevel.PRIVATE) public class TezSetup { - public static ShellResult config(Params params) { + public static ShellResult configure(Params params) { + log.info("Configuring Tez"); TezParams tezParams = (TezParams) params; String confDir = tezParams.confDir(); - String hdfsUser = LocalSettings.users().get("hdfs"); + String serviceHome = tezParams.serviceHome(); + String libDir = serviceHome + "/lib"; + String hadoopUser = LocalSettings.users().get("hadoop"); String tezUser = tezParams.user(); String tezGroup = tezParams.group(); - // tez-site - log.info("Generating [{}/tez-site.xml] file", confDir); + LinuxFileUtils.createDirectories(serviceHome, tezUser, tezGroup, Constants.PERMISSION_755, true); + LinuxFileUtils.createDirectories(confDir, tezUser, tezGroup, Constants.PERMISSION_755, true); + LinuxFileUtils.createDirectories(libDir, tezUser, tezGroup, Constants.PERMISSION_755, true); + + List packages = params.packages(); + for (PackageInfo packageInfo : packages) { + String source = tezParams.stackHome() + "/" + packageInfo.getName(); + String dest = libDir + "/tez.tar.gz"; + LinuxFileUtils.copyFile(source, dest); + LinuxFileUtils.updateOwner(dest, tezUser, tezGroup, false); + } + LinuxFileUtils.toFile( ConfigType.XML, MessageFormat.format("{0}/tez-site.xml", confDir), @@ -57,8 +71,6 @@ public static ShellResult config(Params params) { tezParams.tezSite(), tezParams.getGlobalParamsMap()); - // tez-env - log.info("Generating [{}/tez-env.sh] file", confDir); LinuxFileUtils.toFileByTemplate( tezParams.getTezEnvContent(), MessageFormat.format("{0}/tez-env.sh", confDir), @@ -67,10 +79,10 @@ public static ShellResult config(Params params) { PERMISSION_755, tezParams.getGlobalParamsMap()); - HdfsUtil.createDirectory(hdfsUser, "/apps"); - HdfsUtil.uploadFile(tezUser, tezParams.serviceHome() + "/lib/tez.tar.gz", "/apps/tez"); + // HdfsUtil.createDirectory(hadoopUser, "/apps"); + // HdfsUtil.uploadFile(tezUser, tezParams.serviceHome() + "/lib/tez.tar.gz", "/apps/tez"); log.info("Successfully configured Tez"); - return ShellResult.success("Tez Configure success!"); + return ShellResult.success(); } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java index 188279178..b0ca3d530 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java @@ -18,6 +18,9 @@ */ package org.apache.bigtop.manager.stack.core.tarball; +import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo; +import org.apache.bigtop.manager.stack.core.exception.StackException; + import lombok.extern.slf4j.Slf4j; import java.io.File; @@ -30,7 +33,55 @@ public class TarballDownloader { @SuppressWarnings("ResultOfMethodCallIgnored") - public static Boolean downloadFile(String fileUrl, String saveDir) { + public static void download(String remoteUrl, String saveDir, PackageInfo packageInfo) { + File localFile = new File(saveDir + File.separator + packageInfo.getName()); + String algorithm = packageInfo.getChecksum().split(":")[0]; + String checksum = packageInfo.getChecksum().split(":")[1]; + + if (localFile.exists()) { + log.info("File [{}] exists, validating checksum", localFile.getAbsolutePath()); + } else { + log.info("Downloading [{}] to [{}]", remoteUrl, saveDir); + download(remoteUrl, saveDir); + } + + boolean validateChecksum = ChecksumValidator.validateChecksum(algorithm, checksum, localFile); + if (!validateChecksum) { + log.warn("Invalid checksum for [{}], re-downloading...", localFile.getAbsolutePath()); + localFile.delete(); + download(remoteUrl, saveDir); + } + + validateChecksum = ChecksumValidator.validateChecksum(algorithm, checksum, localFile); + if (!validateChecksum) { + log.error("Invalid checksum for [{}], exiting...", localFile.getAbsolutePath()); + throw new StackException("Invalid checksum for " + localFile.getAbsolutePath()); + } + + log.info("Checksum validate successfully for [{}]", localFile.getAbsolutePath()); + } + + private static void download(String remoteUrl, String saveDir) { + int i = 1; + while (true) { + Boolean downloaded = downloadFile(remoteUrl, saveDir); + if (downloaded) { + break; + } else { + if (i == 3) { + log.error("Failed to download [{}], exiting...", remoteUrl); + throw new StackException("Failed to download " + remoteUrl); + } else { + log.error("Failed to download [{}], retrying...: {}", remoteUrl, i); + } + } + + i++; + } + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + private static Boolean downloadFile(String fileUrl, String saveDir) { HttpURLConnection httpConn = null; try { URL url = new URL(fileUrl); diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java index ef3c9f7a5..f0d99c50d 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java @@ -126,7 +126,7 @@ private static void extractTarGz(File tarball, Function builderParameters = new ArrayList<>(); + builderParameters.add("cp"); + if (Files.exists(Path.of(source)) && Files.isDirectory(Paths.get(source))) { + builderParameters.add("-r"); + } + + builderParameters.add(source); + builderParameters.add(dest); + + try { + ShellResult shellResult = sudoExecCmd(builderParameters); + if (shellResult.getExitCode() != MessageConstants.SUCCESS_CODE) { + throw new StackException(shellResult.getErrMsg()); + } + } catch (IOException e) { + throw new StackException(e); + } + } + public static void moveFile(String source, String dest) { if (StringUtils.isBlank(source) || StringUtils.isBlank(dest)) { log.error("source and dest must not be empty");