From a7f4b35755f475fd44b2a180a706b69aef63d4cb Mon Sep 17 00:00:00 2001 From: Zhiguo Wu Date: Tue, 10 Dec 2024 15:11:11 +0800 Subject: [PATCH] BIGTOP-4301: Install Solr with tarball --- .../solr/configuration/solr-log4j.xml | 103 ------------------ .../services/solr/configuration/solr-xml.xml | 102 +++-------------- .../{solr-env.xml => solr.in.xml} | 28 ++--- .../bigtop/v3_3_0/solr/SolrClientScript.java | 42 ------- ...verScript.java => SolrInstanceScript.java} | 37 ++++--- .../stack/bigtop/v3_3_0/solr/SolrParams.java | 48 ++++---- .../stack/bigtop/v3_3_0/solr/SolrSetup.java | 24 ++-- 7 files changed, 76 insertions(+), 308 deletions(-) delete mode 100644 bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/configuration/solr-log4j.xml rename bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/configuration/{solr-env.xml => solr.in.xml} (93%) delete mode 100644 bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrClientScript.java rename bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/{SolrServerScript.java => SolrInstanceScript.java} (72%) diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/configuration/solr-log4j.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/configuration/solr-log4j.xml deleted file mode 100644 index e8483c49..00000000 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/configuration/solr-log4j.xml +++ /dev/null @@ -1,103 +0,0 @@ - - - - - log_maxfilesize - 10 - The maximum size of backup file before the log is rotated - Solr Log: backup file size - - - - log_maxbackupindex - 9 - The number of backup files - Solr Log: # of backup files - - - content - solr-log4j template - This is the freemarker template for log4j2.xml - - longtext - - - - - - - - %d{ISO8601} [%t] %-5p [%X{collection} %X{shard} %X{replica} %X{core}] %C (%F:%L) - %m%n - - - - - - - - %d{ISO8601} [%t] %-5p [%X{collection} %X{shard} %X{replica} %X{core}] %C (%F:%L) - %m%n - - - - - - - - - - - - - %d{ISO8601} [%t] %-5p [%X{collection} %X{shard} %X{replica} %X{core}] %C (%F:%L) - %m%n - - - - - - - - - - - - - - - - - - - - - - - - - ]]> - - - diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/configuration/solr-xml.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/configuration/solr-xml.xml index 856ae587..a255e9b5 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/configuration/solr-xml.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/configuration/solr-xml.xml @@ -18,74 +18,6 @@ ~ under the License. --> - - maxBooleanClauses - ${solr.max.booleanClauses:1024} - - - sharedLib - ${solr.sharedLib:} - - - allowPaths - ${solr.allowPaths:} - - - host - ${host:} - - - hostPort - ${solr.port.advertise:0} - - - hostContext - ${hostContext:solr} - - - genericCoreNodeNames - ${genericCoreNodeNames:true} - - - zkClientTimeout - ${zkClientTimeout:30000} - - - distribUpdateSoTimeout - ${distribUpdateSoTimeout:600000} - - - distribUpdateConnTimeout - ${distribUpdateConnTimeout:60000} - - - zkCredentialsProvider - ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} - - - zkCredentialsProvider - ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} - - - zkACLProvider - ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} - - - socketTimeout - ${socketTimeout:600000} - - - connTimeout - ${connTimeout:60000} - - - shardsWhitelist - ${solr.shardsWhitelist:} - - - metricsenabled - {metricsEnabled:true - content solr-xml template @@ -94,29 +26,31 @@ longtext - ${maxBooleanClauses} - ${sharedLib} - ${allowPaths} + ${solr.max.booleanClauses:1024} + ${solr.sharedLib:} + ${solr.allowPaths:} - ${host} - ${hostPort} - ${hostContext} - ${genericCoreNodeNames} - ${zkClientTimeout} - ${distribUpdateSoTimeout} - ${distribUpdateConnTimeout} - ${zkCredentialsProvider} - ${zkACLProvider} + ${host:} + ${solr.port.advertise:0} + ${hostContext:solr} + ${genericCoreNodeNames:true} + ${zkClientTimeout:30000} + ${distribUpdateSoTimeout:600000} + ${distribUpdateConnTimeout:60000} + ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} + ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} - ${socketTimeout} - ${connTimeout} - ${shardsWhitelist} + ${socketTimeout:600000} + ${connTimeout:60000} + ${solr.shardsWhitelist:} - + + ]]> diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/configuration/solr-env.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/configuration/solr.in.xml similarity index 93% rename from bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/configuration/solr-env.xml rename to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/configuration/solr.in.xml index 39657e63..405ac514 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/configuration/solr-env.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/configuration/solr.in.xml @@ -42,18 +42,6 @@ Directory for Solr logs - - solr_user - solr - Solr user - - - - solr_datadir - /var/lib/solr/data - Directory for storting Solr index. Make sure you have enough disk space - - solr_ssl_enabled false @@ -199,7 +187,7 @@ solr_extra_java_opts - + Extra Solr java options (e.g.: -Dproperty=value), that will be added to SOLR_OPTS environment variable @@ -207,11 +195,11 @@ solr_gc_log_opts - + solr_gc_tune - + solr_min_mem @@ -226,11 +214,11 @@ false - + content - solr-env template - This is the freemarker template for solr-env.sh file + solr.in template + This is the freemarker template for solr.in.sh file longtext @@ -288,12 +276,12 @@ SOLR_PID_DIR=${solr_pid_dir} # Path to a directory where Solr creates index files, the specified directory # must contain a solr.xml; by default, Solr will use server/solr -SOLR_HOME=${solr_datadir} +SOLR_HOME=${solr_home}/server/solr # Solr provides a default Log4J configuration properties file in server/resources # however, you may want to customize the log settings and file appender location # so you can point the script to use a different log4j.properties file -LOG4J_PROPS=/etc/solr/conf/log4j2.xml +LOG4J_PROPS=${solr_home}/server/resources/log4j2.xml # Location where Solr should write logs to; should agree with the file appender # settings in server/resources/log4j.properties diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrClientScript.java deleted file mode 100644 index e8997d22..00000000 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrClientScript.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bigtop.manager.stack.bigtop.v3_3_0.solr; - -import org.apache.bigtop.manager.common.shell.ShellResult; -import org.apache.bigtop.manager.stack.core.spi.param.Params; -import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript; -import org.apache.bigtop.manager.stack.core.spi.script.Script; - -import com.google.auto.service.AutoService; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@AutoService(Script.class) -public class SolrClientScript extends AbstractClientScript { - - @Override - public ShellResult configure(Params params) { - return SolrSetup.config(params); - } - - @Override - public String getComponentName() { - return "solr_client"; - } -} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrInstanceScript.java similarity index 72% rename from bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrServerScript.java rename to bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrInstanceScript.java index 2e74824f..0ab3d0ca 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrInstanceScript.java @@ -18,6 +18,7 @@ */ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.solr; +import org.apache.bigtop.manager.common.constants.MessageConstants; import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.stack.core.exception.StackException; import org.apache.bigtop.manager.stack.core.spi.param.Params; @@ -30,25 +31,33 @@ import java.io.IOException; import java.text.MessageFormat; +import java.util.Properties; @Slf4j @AutoService(Script.class) -public class SolrServerScript extends AbstractServerScript { +public class SolrInstanceScript extends AbstractServerScript { + + @Override + public ShellResult add(Params params) { + Properties properties = new Properties(); + properties.setProperty(PROPERTY_KEY_SKIP_LEVELS, "1"); + + return super.add(params, properties); + } @Override public ShellResult configure(Params params) { - return SolrSetup.config(params); + return SolrSetup.configure(params); } @Override public ShellResult start(Params params) { configure(params); + createZNode(params); SolrParams solrParams = (SolrParams) params; - String confdir = MessageFormat.format("export SOLR_INCLUDE={0}/solr-env.sh", solrParams.confDir()); - ShellResult shellResult = create_solr_znode(params); String cmd = MessageFormat.format( - "{0} && {1}/bin/solr start -cloud -noprompt -s {2} -z {3}", - confdir, solrParams.serviceHome(), solrParams.getSolrDataDir(), solrParams.zkHost()); + "{0}/bin/solr start -cloud -noprompt -s {0}/server/solr -z {1}", + solrParams.serviceHome(), solrParams.getZkHost()); try { return LinuxOSUtils.sudoExecCmd(cmd, solrParams.user()); } catch (IOException e) { @@ -73,15 +82,17 @@ public ShellResult status(Params params) { return LinuxOSUtils.checkProcess(solrParams.getSolrPidFile()); } - public ShellResult create_solr_znode(Params params) { - configure(params); + public void createZNode(Params params) { SolrParams solrParams = (SolrParams) params; - String zNode = solrParams.getZnode(); - String solrZkString = solrParams.getzkString(); + String zNode = solrParams.getZNode(); + String zkString = solrParams.getZkString(); String cmd = MessageFormat.format( - "{0}/bin/solr zk mkroot {1} -z {2} 2>&1", solrParams.serviceHome(), zNode, solrZkString); + "{0}/bin/solr zk mkroot {1} -z {2} 2>&1", solrParams.serviceHome(), zNode, zkString); try { - return LinuxOSUtils.sudoExecCmd(cmd, solrParams.user()); + ShellResult shellResult = LinuxOSUtils.sudoExecCmd(cmd, solrParams.user()); + if (shellResult.getExitCode() != MessageConstants.SUCCESS_CODE) { + throw new StackException(shellResult.getErrMsg()); + } } catch (IOException e) { throw new StackException(e); } @@ -89,6 +100,6 @@ public ShellResult create_solr_znode(Params params) { @Override public String getComponentName() { - return "solr_server"; + return "solr_instance"; } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrParams.java index 75613a80..1beb8653 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrParams.java @@ -43,14 +43,16 @@ public class SolrParams extends BigtopParams { private String solrPort = "8983"; private String solrLogDir = "/var/log/solr"; private String solrPidDir = "/var/run/solr"; - private String solrHomeDir = "/var/lib/solr"; - private String solrDataDir = "/var/lib/solr/data"; private String solrPidFile = solrPidDir + "/solr-" + solrPort + ".pid"; + private String zNode; + private String zkString; + private String zkHost; + public SolrParams(CommandPayload commandPayload) { super(commandPayload); globalParamsMap.put("java_home", Environments.getJavaHome()); - globalParamsMap.put("solr_home", solrHomeDir); + globalParamsMap.put("solr_home", serviceHome()); globalParamsMap.put("security_enabled", false); globalParamsMap.put("solr_pid_file", solrPidFile); globalParamsMap.put("solr_user", user()); @@ -62,40 +64,28 @@ public Map solrXml() { return LocalSettings.configurations(getServiceName(), "solr-xml"); } - @GlobalParams - public Map solrLog4j() { - return LocalSettings.configurations(getServiceName(), "solr-log4j"); - } - - public String getZnode() { - Map solrEnv = LocalSettings.configurations(getServiceName(), "solr-env"); - return (String) solrEnv.get("solr_znode"); - } - - public String getzkString() { - List ZookeeperServerHosts = LocalSettings.hosts("zookeeper_server"); - Map Zkport = LocalSettings.configurations("zookeeper", "zoo.cfg"); - String clientPort = (String) Zkport.get("clientPort"); - return MessageFormat.format("{0}:{1}", ZookeeperServerHosts.get(0), clientPort); - } - - public String zkHost() { - String Znode = getZnode(); - String SolrZkstring = getzkString(); - return MessageFormat.format("{0}{1}", SolrZkstring, Znode); - } - @GlobalParams public Map solrEnv() { - Map solrEnv = LocalSettings.configurations(getServiceName(), "solr-env"); + Map solrEnv = LocalSettings.configurations(getServiceName(), "solr.in"); solrLogDir = (String) solrEnv.get("solr_log_dir"); solrPidDir = (String) solrEnv.get("solr_pid_dir"); - solrDataDir = (String) solrEnv.get("solr_datadir"); - solrPort = (String) solrEnv.get("SOLR_PORT"); + solrPort = (String) solrEnv.get("solr_port"); solrPidFile = solrPidDir + "/solr-" + solrPort + ".pid"; + + List ZookeeperServerHosts = LocalSettings.hosts("zookeeper_server"); + Map ZKPort = LocalSettings.configurations("zookeeper", "zoo.cfg"); + String clientPort = (String) ZKPort.get("clientPort"); + zNode = (String) solrEnv.get("solr_znode"); + zkString = MessageFormat.format("{0}:{1}", ZookeeperServerHosts.get(0), clientPort); + zkHost = MessageFormat.format("{0}{1}", zkString, zNode); return solrEnv; } + @Override + public String confDir() { + return serviceHome() + "/server/solr"; + } + @Override public String getServiceName() { return "solr"; diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrSetup.java index 9bc57742..e4543a25 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrSetup.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrSetup.java @@ -39,8 +39,8 @@ @NoArgsConstructor(access = AccessLevel.PRIVATE) public class SolrSetup { - public static ShellResult config(Params params) { - log.info("solrSetup config"); + public static ShellResult configure(Params params) { + log.info("Configuring Solr"); SolrParams solrParams = (SolrParams) params; String confDir = solrParams.confDir(); @@ -48,41 +48,31 @@ public static ShellResult config(Params params) { String solrGroup = solrParams.group(); Map solrEnv = solrParams.solrEnv(); - LinuxFileUtils.createDirectories(solrParams.getSolrHomeDir(), solrUser, solrGroup, PERMISSION_755, true); - LinuxFileUtils.createDirectories(solrParams.getSolrDataDir(), solrUser, solrGroup, PERMISSION_755, true); LinuxFileUtils.createDirectories(solrParams.getSolrLogDir(), solrUser, solrGroup, PERMISSION_755, true); LinuxFileUtils.createDirectories(solrParams.getSolrPidDir(), solrUser, solrGroup, PERMISSION_755, true); - // solr-env.xml List zookeeperServerHosts = LocalSettings.hosts("zookeeper_server"); Map paramMap = new HashMap<>(); paramMap.put("zookeeper_quorum", zookeeperServerHosts); paramMap.put("host", solrParams.hostname()); LinuxFileUtils.toFileByTemplate( solrEnv.get("content").toString(), - MessageFormat.format("{0}/solr-env.sh", confDir), + MessageFormat.format("{0}/solr.in.sh", solrParams.serviceHome() + "/bin"), solrUser, solrGroup, Constants.PERMISSION_755, solrParams.getGlobalParamsMap(), paramMap); - // solr-log4j.xml - LinuxFileUtils.toFileByTemplate( - solrParams.solrLog4j().get("content").toString(), - MessageFormat.format("{0}/log4j2.xml", confDir), - solrUser, - solrGroup, - Constants.PERMISSION_755, - solrParams.getGlobalParamsMap()); - // solr-xml.xml + LinuxFileUtils.toFileByTemplate( solrParams.solrXml().get("content").toString(), - MessageFormat.format("{0}/solr.xml", solrParams.getSolrDataDir()), + MessageFormat.format("{0}/solr.xml", confDir), solrUser, solrGroup, Constants.PERMISSION_755, solrParams.getGlobalParamsMap()); - return ShellResult.success("solr Server Configure success!"); + log.info("Successfully configured Solr"); + return ShellResult.success(); } }