diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/metainfo.xml
index dd989a05e..6fc6bbd49 100644
--- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/metainfo.xml
+++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/metainfo.xml
@@ -52,17 +52,19 @@
x86_64
+ aarch64
- flink_3_3_0
+ flink-1.16.2-1.tgz
+ SHA-256:8c6da7cbab14b9cf0ce39c8c873b29f16c83df0b95b7af5cf3061a468efac228
- yarn
+ hadoop
\ No newline at end of file
diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml
index ce0615be0..0ced8143c 100644
--- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml
+++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml
@@ -112,7 +112,7 @@
hadoop-3.3.6-1.tar.gz
- SHA-256:2a9312eac69a542c6266f1689d29ded243dde2d61add333a9762829008f97442
+ SHA-256:0da6364126c6ea6dd0f5b545afec33957ee801d6db6c0874edb0aa5be1e89995
diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml
index 2c113d538..75ed8e854 100644
--- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml
+++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml
@@ -43,10 +43,12 @@
x86_64
+ aarch64
- kafka_3_3_0
+ kafka-2.8.2-1.tgz
+ SHA-256:30a2d69ef081813624273d8a406c9b803c5868df998484e27207f85ea217870f
diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/metainfo.xml
index 95ee24a02..32c5465f4 100644
--- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/metainfo.xml
+++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/metainfo.xml
@@ -47,10 +47,12 @@
x86_64
+ aarch64
- solr_3_3_0
+ solr-8.11.2-2.tgz
+ SHA-256:fba6618e4c3a3133a4c4d57718e52cc797d10374f056825329a9e76b93de9f48
diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml
index 3ca392610..509db8315 100644
--- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml
+++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml
@@ -39,17 +39,19 @@
x86_64
+ aarch64
- tez_3_3_0
+ tez-0.10.2-1.tar.gz
+ SHA-256:ebc2c195780fc76fb9b12e8987e48c301ca0b1916dca515f5c7a1122d74f397e
- hdfs
+ hadoop
diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml
index 7b35ebffb..7748ff51c 100644
--- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml
+++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml
@@ -54,7 +54,7 @@
zookeeper-3.7.2-1.tar.gz
- SHA-256:380f15d55c0282e33fdc7c2ec551bc5586f0ac126c243cd84347ccb775b846f3
+ SHA-256:83e07f914eb3477c77245f4dc44031b82ea6ef1be3691687f469ddc4b8c720bb
diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java
index e65478b7f..bcbea7d92 100644
--- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java
+++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java
@@ -18,21 +18,49 @@
*/
package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez;
+import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo;
+import org.apache.bigtop.manager.common.message.entity.pojo.RepoInfo;
import org.apache.bigtop.manager.common.shell.ShellResult;
import org.apache.bigtop.manager.stack.core.spi.param.Params;
import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript;
import org.apache.bigtop.manager.stack.core.spi.script.Script;
+import org.apache.bigtop.manager.stack.core.tarball.TarballDownloader;
import com.google.auto.service.AutoService;
import lombok.extern.slf4j.Slf4j;
+import java.io.File;
+import java.util.List;
+
@Slf4j
@AutoService(Script.class)
public class TezClientScript extends AbstractClientScript {
+ /**
+ * Tez tarball file doesn't need to be extracted, so here we override {@link #add(Params)} method
+ * and do nothing after download the tarball file.
+ *
+ * @param params the parameters required for installation
+ * @return ShellResult
+ */
+ @Override
+ public ShellResult add(Params params) {
+ RepoInfo repo = params.repo();
+ List packages = params.packages();
+
+ String repoUrl = repo.getBaseUrl();
+ String stackHome = params.stackHome();
+ for (PackageInfo packageInfo : packages) {
+ String remoteUrl = repoUrl + File.separator + packageInfo.getName();
+ TarballDownloader.download(remoteUrl, stackHome, packageInfo);
+ }
+
+ return ShellResult.success();
+ }
+
@Override
public ShellResult configure(Params params) {
- return TezSetup.config(params);
+ return TezSetup.configure(params);
}
@Override
diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java
index 25995737c..3d2c87846 100644
--- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java
+++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java
@@ -77,27 +77,14 @@ public Map tezEnv() {
return tezEnv;
}
- @Override
- public String confDir() {
- return "/etc/tez/conf";
- }
-
public String hadoopConfDir() {
- return "/etc/hadoop/conf";
+ return hadoopHome() + "/etc/hadoop";
}
public String hadoopHome() {
return stackHome() + "/hadoop";
}
- public String hdfsHome() {
- return stackHome() + "/hadoop-hdfs";
- }
-
- public String yarnHome() {
- return stackHome() + "/hadoop-yarn";
- }
-
@Override
public String getServiceName() {
return "tez";
diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java
index cd995251e..b68c77ad6 100644
--- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java
+++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java
@@ -19,8 +19,8 @@
package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez;
import org.apache.bigtop.manager.common.constants.Constants;
+import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo;
import org.apache.bigtop.manager.common.shell.ShellResult;
-import org.apache.bigtop.manager.stack.bigtop.utils.HdfsUtil;
import org.apache.bigtop.manager.stack.core.enums.ConfigType;
import org.apache.bigtop.manager.stack.core.spi.param.Params;
import org.apache.bigtop.manager.stack.core.utils.LocalSettings;
@@ -31,6 +31,7 @@
import lombok.extern.slf4j.Slf4j;
import java.text.MessageFormat;
+import java.util.List;
import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_755;
@@ -38,16 +39,29 @@
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class TezSetup {
- public static ShellResult config(Params params) {
+ public static ShellResult configure(Params params) {
+ log.info("Configuring Tez");
TezParams tezParams = (TezParams) params;
String confDir = tezParams.confDir();
- String hdfsUser = LocalSettings.users().get("hdfs");
+ String serviceHome = tezParams.serviceHome();
+ String libDir = serviceHome + "/lib";
+ String hadoopUser = LocalSettings.users().get("hadoop");
String tezUser = tezParams.user();
String tezGroup = tezParams.group();
- // tez-site
- log.info("Generating [{}/tez-site.xml] file", confDir);
+ LinuxFileUtils.createDirectories(serviceHome, tezUser, tezGroup, Constants.PERMISSION_755, true);
+ LinuxFileUtils.createDirectories(confDir, tezUser, tezGroup, Constants.PERMISSION_755, true);
+ LinuxFileUtils.createDirectories(libDir, tezUser, tezGroup, Constants.PERMISSION_755, true);
+
+ List packages = params.packages();
+ for (PackageInfo packageInfo : packages) {
+ String source = tezParams.stackHome() + "/" + packageInfo.getName();
+ String dest = libDir + "/tez.tar.gz";
+ LinuxFileUtils.copyFile(source, dest);
+ LinuxFileUtils.updateOwner(dest, tezUser, tezGroup, false);
+ }
+
LinuxFileUtils.toFile(
ConfigType.XML,
MessageFormat.format("{0}/tez-site.xml", confDir),
@@ -57,8 +71,6 @@ public static ShellResult config(Params params) {
tezParams.tezSite(),
tezParams.getGlobalParamsMap());
- // tez-env
- log.info("Generating [{}/tez-env.sh] file", confDir);
LinuxFileUtils.toFileByTemplate(
tezParams.getTezEnvContent(),
MessageFormat.format("{0}/tez-env.sh", confDir),
@@ -67,10 +79,10 @@ public static ShellResult config(Params params) {
PERMISSION_755,
tezParams.getGlobalParamsMap());
- HdfsUtil.createDirectory(hdfsUser, "/apps");
- HdfsUtil.uploadFile(tezUser, tezParams.serviceHome() + "/lib/tez.tar.gz", "/apps/tez");
+ // HdfsUtil.createDirectory(hadoopUser, "/apps");
+ // HdfsUtil.uploadFile(tezUser, tezParams.serviceHome() + "/lib/tez.tar.gz", "/apps/tez");
log.info("Successfully configured Tez");
- return ShellResult.success("Tez Configure success!");
+ return ShellResult.success();
}
}
diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java
index 188279178..b0ca3d530 100644
--- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java
+++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java
@@ -18,6 +18,9 @@
*/
package org.apache.bigtop.manager.stack.core.tarball;
+import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo;
+import org.apache.bigtop.manager.stack.core.exception.StackException;
+
import lombok.extern.slf4j.Slf4j;
import java.io.File;
@@ -30,7 +33,55 @@
public class TarballDownloader {
@SuppressWarnings("ResultOfMethodCallIgnored")
- public static Boolean downloadFile(String fileUrl, String saveDir) {
+ public static void download(String remoteUrl, String saveDir, PackageInfo packageInfo) {
+ File localFile = new File(saveDir + File.separator + packageInfo.getName());
+ String algorithm = packageInfo.getChecksum().split(":")[0];
+ String checksum = packageInfo.getChecksum().split(":")[1];
+
+ if (localFile.exists()) {
+ log.info("File [{}] exists, validating checksum", localFile.getAbsolutePath());
+ } else {
+ log.info("Downloading [{}] to [{}]", remoteUrl, saveDir);
+ download(remoteUrl, saveDir);
+ }
+
+ boolean validateChecksum = ChecksumValidator.validateChecksum(algorithm, checksum, localFile);
+ if (!validateChecksum) {
+ log.warn("Invalid checksum for [{}], re-downloading...", localFile.getAbsolutePath());
+ localFile.delete();
+ download(remoteUrl, saveDir);
+ }
+
+ validateChecksum = ChecksumValidator.validateChecksum(algorithm, checksum, localFile);
+ if (!validateChecksum) {
+ log.error("Invalid checksum for [{}], exiting...", localFile.getAbsolutePath());
+ throw new StackException("Invalid checksum for " + localFile.getAbsolutePath());
+ }
+
+ log.info("Checksum validate successfully for [{}]", localFile.getAbsolutePath());
+ }
+
+ private static void download(String remoteUrl, String saveDir) {
+ int i = 1;
+ while (true) {
+ Boolean downloaded = downloadFile(remoteUrl, saveDir);
+ if (downloaded) {
+ break;
+ } else {
+ if (i == 3) {
+ log.error("Failed to download [{}], exiting...", remoteUrl);
+ throw new StackException("Failed to download " + remoteUrl);
+ } else {
+ log.error("Failed to download [{}], retrying...: {}", remoteUrl, i);
+ }
+ }
+
+ i++;
+ }
+ }
+
+ @SuppressWarnings("ResultOfMethodCallIgnored")
+ private static Boolean downloadFile(String fileUrl, String saveDir) {
HttpURLConnection httpConn = null;
try {
URL url = new URL(fileUrl);
diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java
index ef3c9f7a5..f0d99c50d 100644
--- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java
+++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java
@@ -126,7 +126,7 @@ private static void extractTarGz(File tarball, Function builderParameters = new ArrayList<>();
+ builderParameters.add("cp");
+ if (Files.exists(Path.of(source)) && Files.isDirectory(Paths.get(source))) {
+ builderParameters.add("-r");
+ }
+
+ builderParameters.add(source);
+ builderParameters.add(dest);
+
+ try {
+ ShellResult shellResult = sudoExecCmd(builderParameters);
+ if (shellResult.getExitCode() != MessageConstants.SUCCESS_CODE) {
+ throw new StackException(shellResult.getErrMsg());
+ }
+ } catch (IOException e) {
+ throw new StackException(e);
+ }
+ }
+
public static void moveFile(String source, String dest) {
if (StringUtils.isBlank(source) || StringUtils.isBlank(dest)) {
log.error("source and dest must not be empty");