Skip to content

Commit

Permalink
BIGTOP-4297: Install Tez with tarball (#120)
Browse files Browse the repository at this point in the history
  • Loading branch information
kevinw66 authored Dec 9, 2024
1 parent 10425e7 commit faa3f19
Show file tree
Hide file tree
Showing 13 changed files with 147 additions and 80 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -52,17 +52,19 @@
<package-specific>
<architectures>
<arch>x86_64</arch>
<arch>aarch64</arch>
</architectures>
<packages>
<package>
<name>flink_3_3_0</name>
<name>flink-1.16.2-1.tgz</name>
<checksum>SHA-256:8c6da7cbab14b9cf0ce39c8c873b29f16c83df0b95b7af5cf3061a468efac228</checksum>
</package>
</packages>
</package-specific>
</package-specifics>

<required-services>
<service>yarn</service>
<service>hadoop</service>
</required-services>
</service>
</metainfo>
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@
<packages>
<package>
<name>hadoop-3.3.6-1.tar.gz</name>
<checksum>SHA-256:2a9312eac69a542c6266f1689d29ded243dde2d61add333a9762829008f97442</checksum>
<checksum>SHA-256:0da6364126c6ea6dd0f5b545afec33957ee801d6db6c0874edb0aa5be1e89995</checksum>
</package>
</packages>
</package-specific>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,12 @@
<package-specific>
<architectures>
<arch>x86_64</arch>
<arch>aarch64</arch>
</architectures>
<packages>
<package>
<name>kafka_3_3_0</name>
<name>kafka-2.8.2-1.tgz</name>
<checksum>SHA-256:30a2d69ef081813624273d8a406c9b803c5868df998484e27207f85ea217870f</checksum>
</package>
</packages>
</package-specific>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,12 @@
<package-specific>
<architectures>
<arch>x86_64</arch>
<arch>aarch64</arch>
</architectures>
<packages>
<package>
<name>solr_3_3_0</name>
<name>solr-8.11.2-2.tgz</name>
<checksum>SHA-256:fba6618e4c3a3133a4c4d57718e52cc797d10374f056825329a9e76b93de9f48</checksum>
</package>
</packages>
</package-specific>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,17 +39,19 @@
<package-specific>
<architectures>
<arch>x86_64</arch>
<arch>aarch64</arch>
</architectures>
<packages>
<package>
<name>tez_3_3_0</name>
<name>tez-0.10.2-1.tar.gz</name>
<checksum>SHA-256:ebc2c195780fc76fb9b12e8987e48c301ca0b1916dca515f5c7a1122d74f397e</checksum>
</package>
</packages>
</package-specific>
</package-specifics>

<required-services>
<service>hdfs</service>
<service>hadoop</service>
</required-services>
</service>
</metainfo>
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
<packages>
<package>
<name>zookeeper-3.7.2-1.tar.gz</name>
<checksum>SHA-256:380f15d55c0282e33fdc7c2ec551bc5586f0ac126c243cd84347ccb775b846f3</checksum>
<checksum>SHA-256:83e07f914eb3477c77245f4dc44031b82ea6ef1be3691687f469ddc4b8c720bb</checksum>
</package>
</packages>
</package-specific>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,49 @@
*/
package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez;

import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo;
import org.apache.bigtop.manager.common.message.entity.pojo.RepoInfo;
import org.apache.bigtop.manager.common.shell.ShellResult;
import org.apache.bigtop.manager.stack.core.spi.param.Params;
import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript;
import org.apache.bigtop.manager.stack.core.spi.script.Script;
import org.apache.bigtop.manager.stack.core.tarball.TarballDownloader;

import com.google.auto.service.AutoService;
import lombok.extern.slf4j.Slf4j;

import java.io.File;
import java.util.List;

@Slf4j
@AutoService(Script.class)
public class TezClientScript extends AbstractClientScript {

/**
* Tez tarball file doesn't need to be extracted, so here we override {@link #add(Params)} method
* and do nothing after download the tarball file.
*
* @param params the parameters required for installation
* @return ShellResult
*/
@Override
public ShellResult add(Params params) {
RepoInfo repo = params.repo();
List<PackageInfo> packages = params.packages();

String repoUrl = repo.getBaseUrl();
String stackHome = params.stackHome();
for (PackageInfo packageInfo : packages) {
String remoteUrl = repoUrl + File.separator + packageInfo.getName();
TarballDownloader.download(remoteUrl, stackHome, packageInfo);
}

return ShellResult.success();
}

@Override
public ShellResult configure(Params params) {
return TezSetup.config(params);
return TezSetup.configure(params);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,27 +77,14 @@ public Map<String, Object> tezEnv() {
return tezEnv;
}

@Override
public String confDir() {
return "/etc/tez/conf";
}

public String hadoopConfDir() {
return "/etc/hadoop/conf";
return hadoopHome() + "/etc/hadoop";
}

public String hadoopHome() {
return stackHome() + "/hadoop";
}

public String hdfsHome() {
return stackHome() + "/hadoop-hdfs";
}

public String yarnHome() {
return stackHome() + "/hadoop-yarn";
}

@Override
public String getServiceName() {
return "tez";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez;

import org.apache.bigtop.manager.common.constants.Constants;
import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo;
import org.apache.bigtop.manager.common.shell.ShellResult;
import org.apache.bigtop.manager.stack.bigtop.utils.HdfsUtil;
import org.apache.bigtop.manager.stack.core.enums.ConfigType;
import org.apache.bigtop.manager.stack.core.spi.param.Params;
import org.apache.bigtop.manager.stack.core.utils.LocalSettings;
Expand All @@ -31,23 +31,37 @@
import lombok.extern.slf4j.Slf4j;

import java.text.MessageFormat;
import java.util.List;

import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_755;

@Slf4j
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class TezSetup {

public static ShellResult config(Params params) {
public static ShellResult configure(Params params) {
log.info("Configuring Tez");
TezParams tezParams = (TezParams) params;

String confDir = tezParams.confDir();
String hdfsUser = LocalSettings.users().get("hdfs");
String serviceHome = tezParams.serviceHome();
String libDir = serviceHome + "/lib";
String hadoopUser = LocalSettings.users().get("hadoop");
String tezUser = tezParams.user();
String tezGroup = tezParams.group();

// tez-site
log.info("Generating [{}/tez-site.xml] file", confDir);
LinuxFileUtils.createDirectories(serviceHome, tezUser, tezGroup, Constants.PERMISSION_755, true);
LinuxFileUtils.createDirectories(confDir, tezUser, tezGroup, Constants.PERMISSION_755, true);
LinuxFileUtils.createDirectories(libDir, tezUser, tezGroup, Constants.PERMISSION_755, true);

List<PackageInfo> packages = params.packages();
for (PackageInfo packageInfo : packages) {
String source = tezParams.stackHome() + "/" + packageInfo.getName();
String dest = libDir + "/tez.tar.gz";
LinuxFileUtils.copyFile(source, dest);
LinuxFileUtils.updateOwner(dest, tezUser, tezGroup, false);
}

LinuxFileUtils.toFile(
ConfigType.XML,
MessageFormat.format("{0}/tez-site.xml", confDir),
Expand All @@ -57,8 +71,6 @@ public static ShellResult config(Params params) {
tezParams.tezSite(),
tezParams.getGlobalParamsMap());

// tez-env
log.info("Generating [{}/tez-env.sh] file", confDir);
LinuxFileUtils.toFileByTemplate(
tezParams.getTezEnvContent(),
MessageFormat.format("{0}/tez-env.sh", confDir),
Expand All @@ -67,10 +79,10 @@ public static ShellResult config(Params params) {
PERMISSION_755,
tezParams.getGlobalParamsMap());

HdfsUtil.createDirectory(hdfsUser, "/apps");
HdfsUtil.uploadFile(tezUser, tezParams.serviceHome() + "/lib/tez.tar.gz", "/apps/tez");
// HdfsUtil.createDirectory(hadoopUser, "/apps");
// HdfsUtil.uploadFile(tezUser, tezParams.serviceHome() + "/lib/tez.tar.gz", "/apps/tez");

log.info("Successfully configured Tez");
return ShellResult.success("Tez Configure success!");
return ShellResult.success();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
*/
package org.apache.bigtop.manager.stack.core.tarball;

import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo;
import org.apache.bigtop.manager.stack.core.exception.StackException;

import lombok.extern.slf4j.Slf4j;

import java.io.File;
Expand All @@ -30,7 +33,55 @@
public class TarballDownloader {

@SuppressWarnings("ResultOfMethodCallIgnored")
public static Boolean downloadFile(String fileUrl, String saveDir) {
public static void download(String remoteUrl, String saveDir, PackageInfo packageInfo) {
File localFile = new File(saveDir + File.separator + packageInfo.getName());
String algorithm = packageInfo.getChecksum().split(":")[0];
String checksum = packageInfo.getChecksum().split(":")[1];

if (localFile.exists()) {
log.info("File [{}] exists, validating checksum", localFile.getAbsolutePath());
} else {
log.info("Downloading [{}] to [{}]", remoteUrl, saveDir);
download(remoteUrl, saveDir);
}

boolean validateChecksum = ChecksumValidator.validateChecksum(algorithm, checksum, localFile);
if (!validateChecksum) {
log.warn("Invalid checksum for [{}], re-downloading...", localFile.getAbsolutePath());
localFile.delete();
download(remoteUrl, saveDir);
}

validateChecksum = ChecksumValidator.validateChecksum(algorithm, checksum, localFile);
if (!validateChecksum) {
log.error("Invalid checksum for [{}], exiting...", localFile.getAbsolutePath());
throw new StackException("Invalid checksum for " + localFile.getAbsolutePath());
}

log.info("Checksum validate successfully for [{}]", localFile.getAbsolutePath());
}

private static void download(String remoteUrl, String saveDir) {
int i = 1;
while (true) {
Boolean downloaded = downloadFile(remoteUrl, saveDir);
if (downloaded) {
break;
} else {
if (i == 3) {
log.error("Failed to download [{}], exiting...", remoteUrl);
throw new StackException("Failed to download " + remoteUrl);
} else {
log.error("Failed to download [{}], retrying...: {}", remoteUrl, i);
}
}

i++;
}
}

@SuppressWarnings("ResultOfMethodCallIgnored")
private static Boolean downloadFile(String fileUrl, String saveDir) {
HttpURLConnection httpConn = null;
try {
URL url = new URL(fileUrl);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ private static void extractTarGz(File tarball, Function<TarArchiveInputStream, B
}

private static boolean isTarGz(String filePath) {
return filePath.endsWith(".tar.gz");
return filePath.endsWith(".tar.gz") || filePath.endsWith(".tgz");
}

private static boolean isTar(String filePath) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@
package org.apache.bigtop.manager.stack.core.utils;

import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo;
import org.apache.bigtop.manager.stack.core.exception.StackException;
import org.apache.bigtop.manager.stack.core.tarball.ChecksumValidator;
import org.apache.bigtop.manager.stack.core.tarball.TarballDownloader;
import org.apache.bigtop.manager.stack.core.tarball.TarballExtractor;

Expand All @@ -33,7 +31,6 @@
@Slf4j
public class TarballUtils {

@SuppressWarnings("ResultOfMethodCallIgnored")
public static void installPackage(
String repoUrl, String stackHome, String serviceHome, PackageInfo packageInfo, Integer skipLevels) {
if (Files.exists(Path.of(serviceHome))) {
Expand All @@ -43,51 +40,10 @@ public static void installPackage(

String remoteUrl = repoUrl + File.separator + packageInfo.getName();
File localFile = new File(stackHome + File.separator + packageInfo.getName());
String algorithm = packageInfo.getChecksum().split(":")[0];
String checksum = packageInfo.getChecksum().split(":")[1];
TarballDownloader.download(remoteUrl, stackHome, packageInfo);

if (localFile.exists()) {
log.info("File [{}] exists, validating checksum", localFile.getAbsolutePath());
} else {
log.info("Downloading [{}] to [{}]", remoteUrl, stackHome);
downloadPackage(remoteUrl, stackHome);
}

boolean validateChecksum = ChecksumValidator.validateChecksum(algorithm, checksum, localFile);
if (!validateChecksum) {
log.warn("Invalid checksum for [{}], re-downloading...", localFile.getAbsolutePath());
localFile.delete();
downloadPackage(remoteUrl, stackHome);
}

validateChecksum = ChecksumValidator.validateChecksum(algorithm, checksum, localFile);
if (!validateChecksum) {
log.error("Invalid checksum for [{}], exiting...", localFile.getAbsolutePath());
throw new StackException("Invalid checksum for " + localFile.getAbsolutePath());
}

log.info("Checksum validate successfully for [{}]", localFile.getAbsolutePath());
log.info("Extracting [{}] to [{}]", localFile.getAbsolutePath(), serviceHome);
TarballExtractor.extractTarball(localFile.getAbsolutePath(), serviceHome, skipLevels);
log.info("File [{}] successfully extracted to [{}]", localFile.getAbsolutePath(), serviceHome);
}

private static void downloadPackage(String remoteUrl, String saveDir) {
int i = 1;
while (true) {
Boolean downloaded = TarballDownloader.downloadFile(remoteUrl, saveDir);
if (downloaded) {
break;
} else {
if (i == 3) {
log.error("Failed to download [{}], exiting...", remoteUrl);
throw new StackException("Failed to download " + remoteUrl);
} else {
log.error("Failed to download [{}], retrying...: {}", remoteUrl, i);
}
}

i++;
}
}
}
Loading

0 comments on commit faa3f19

Please sign in to comment.