fix bugs

apache · Dec 7, 2024 · 8ab2c66 · 8ab2c66
1 parent dcdcf6f
commit 8ab2c66
Show file tree

Hide file tree

Showing 16 changed files with 100 additions and 75 deletions.
diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/ServiceDao.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/ServiceDao.java
@@ -34,5 +34,5 @@ public interface ServiceDao extends BaseDao<ServicePO> {
 
     ServicePO findByClusterIdAndName(@Param("clusterId") Long clusterId, @Param("name") String name);
 
-    List<ServicePO> findByClusterIdAndNames(@Param("clusterId") Long clusterId, @Param("name") List<String> names);
+    List<ServicePO> findByClusterIdAndNames(@Param("clusterId") Long clusterId, @Param("names") List<String> names);
 }
diff --git a/...erver/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hadoop-env.xml b/...erver/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hadoop-env.xml
@@ -66,6 +66,8 @@ USER="$(whoami)"
 export JAVA_HOME=${java_home!}
 export HADOOP_HOME=${hadoop_home!}
 export HADOOP_CONF_DIR=${hadoop_conf_dir!}
+export HADOOP_LOG_DIR=${hadoop_log_dir}
+export HADOOP_PID_DIR=${hadoop_pid_dir}
 export HADOOP_HEAPSIZE_MAX=${hadoop_heapsize_max}
 export HADOOP_HEAPSIZE_MIN=${hadoop_heapsize_min}
 
@@ -77,7 +79,7 @@ export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)}
 
 export HADOOP_ROOT_LOGGER=${hadoop_root_logger}
 export HADOOP_LIBEXEC_DIR=${hadoop_libexec_dir}
-
+export LD_LIBRARY_PATH=${hadoop_home!}/lib/native
 ]]>
         </value>
         <attrs>

diff --git a/...server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hdfs-site.xml b/...server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hdfs-site.xml
@@ -323,7 +323,7 @@
     </property>
     <property>
         <name>dfs.domain.socket.path</name>
-        <value>/var/run/hadoop-hdfs/dn._PORT</value>
+        <value>/var/run/hadoop/dn._PORT</value>
         <description>
             This is a path to a UNIX domain socket that will be used for communication between the DataNode and local
             HDFS clients.

diff --git a/...server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/yarn-site.xml b/...server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/yarn-site.xml
@@ -76,7 +76,7 @@
     </property>
     <property>
         <name>yarn.admin.acl</name>
-        <value />
+        <value>hadoop</value>
         <description>ACL of who can be admin of the YARN cluster.</description>
     </property>
     <!-- NodeManager -->
@@ -323,7 +323,7 @@
     </property>
     <property>
         <name>yarn.timeline-service.leveldb-timeline-store.path</name>
-        <value>/var/log/hadoop-yarn/timeline</value>
+        <value>/var/log/hadoop/timeline</value>
         <description>
             Store file name for leveldb timeline store
         </description>
@@ -351,14 +351,14 @@
         </description>
     </property>
     <property>
-        <description>Enable age off of timeline store data.</description>
         <name>yarn.timeline-service.ttl-enable</name>
         <value>true</value>
+        <description>Enable age off of timeline store data.</description>
     </property>
     <property>
-        <description>Time to live for timeline store data in milliseconds.</description>
         <name>yarn.timeline-service.ttl-ms</name>
         <value>2678400000</value>
+        <description>Time to live for timeline store data in milliseconds.</description>
     </property>
     <property>
         <name>yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms</name>
@@ -373,22 +373,7 @@
             true, then yarn.timeline-service.state-store-class must be specified.
         </description>
     </property>
-    <property>
-        <name>yarn.acl.enable</name>
-        <value>false</value>
-        <description>Are acls enabled.</description>
-    </property>
-    <property>
-        <name>yarn.authorization-provider</name>
-        <description>Yarn authorization provider class.</description>
-    </property>
-    <property>
-        <name>yarn.admin.acl</name>
-        <value>yarn</value>
-        <description>ACL of who can be admin of the YARN cluster.</description>
-    </property>
-    <!--ats
-    v1.5 properties-->
+    <!--ats v1.5 properties-->
     <property>
         <name>yarn.timeline-service.store-class</name>
         <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>

diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml
@@ -20,11 +20,11 @@
 
 <metainfo>
     <service>
-        <name>hdfs</name>
-        <display-name>HDFS</display-name>
+        <name>hadoop</name>
+        <display-name>Hadoop</display-name>
         <desc>Apache Hadoop Distributed File System.</desc>
         <version>3.3.6-1</version>
-        <user>hdfs</user>
+        <user>hadoop</user>
 
         <components>
             <!-- HDFS Components -->
@@ -40,9 +40,9 @@
                 <quick-link>
                     <display-name>NameNode UI</display-name>
                     <http-port-property>dfs.namenode.http-address</http-port-property>
-                    <http-port-default>50070</http-port-default>
+                    <http-port-default>9870</http-port-default>
                     <https-port-property>dfs.namenode.http-address</https-port-property>
-                    <https-port-default>50470</https-port-default>
+                    <https-port-default>9870</https-port-default>
                 </quick-link>
             </component>
             <component>

diff --git a/...bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkParams.java b/...bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkParams.java
@@ -69,7 +69,7 @@ public FlinkParams(CommandPayload commandPayload) {
 
     @GlobalParams
     public Map<String, Object> flinkConf() {
-        Map<String, Object> configurations = LocalSettings.configurations(serviceName(), "flink-conf");
+        Map<String, Object> configurations = LocalSettings.configurations(getServiceName(), "flink-conf");
         flinkConfContent = (String) configurations.get("content");
 
         jobManagerArchiveFsDir = (String) configurations.get("jobmanager.archive.fs.dir");
@@ -82,7 +82,7 @@ public Map<String, Object> flinkConf() {
 
     @GlobalParams
     public Map<String, Object> flinkEnv() {
-        Map<String, Object> configurations = LocalSettings.configurations(serviceName(), "flink-env");
+        Map<String, Object> configurations = LocalSettings.configurations(getServiceName(), "flink-env");
         flinkLogDir = (String) configurations.get("flink_log_dir");
         flinkPidDir = (String) configurations.get("flink_pid_dir");
         historyServerPidFile = MessageFormat.format("{0}/flink-{1}-historyserver.pid", flinkPidDir, user());
@@ -91,30 +91,30 @@ public Map<String, Object> flinkEnv() {
 
     @GlobalParams
     public Map<String, Object> flinkLog4jProperties() {
-        Map<String, Object> configurations = LocalSettings.configurations(serviceName(), "flink-log4j-properties");
+        Map<String, Object> configurations = LocalSettings.configurations(getServiceName(), "flink-log4j-properties");
         flinkLog4jPropertiesContent = (String) configurations.get("content");
         return configurations;
     }
 
     @GlobalParams
     public Map<String, Object> flinkLog4jCLiProperties() {
-        Map<String, Object> configurations = LocalSettings.configurations(serviceName(), "flink-log4j-cli-properties");
+        Map<String, Object> configurations = LocalSettings.configurations(getServiceName(), "flink-log4j-cli-properties");
         flinkLog4jCLiPropertiesContent = (String) configurations.get("content");
         return configurations;
     }
 
     @GlobalParams
     public Map<String, Object> flinkLog4jConsoleProperties() {
         Map<String, Object> configurations =
-                LocalSettings.configurations(serviceName(), "flink-log4j-console-properties");
+                LocalSettings.configurations(getServiceName(), "flink-log4j-console-properties");
         flinkLog4jConsolePropertiesContent = (String) configurations.get("content");
         return configurations;
     }
 
     @GlobalParams
     public Map<String, Object> flinkLog4jSessionProperties() {
         Map<String, Object> configurations =
-                LocalSettings.configurations(serviceName(), "flink-log4j-session-properties");
+                LocalSettings.configurations(getServiceName(), "flink-log4j-session-properties");
         flinkLog4jSessionPropertiesContent = (String) configurations.get("content");
         return configurations;
     }
@@ -131,4 +131,9 @@ public String hadoopConfDir() {
     public String hadoopHome() {
         return stackHome() + "/hadoop";
     }
+
+    @Override
+    public String getServiceName() {
+        return "flink";
+    }
 }
diff --git a/...gtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HadoopParams.java b/...gtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HadoopParams.java
@@ -80,23 +80,23 @@ public HadoopParams(CommandPayload commandPayload) {
     }
 
     public String hadoopLimits() {
-        Map<String, Object> hadoopConf = LocalSettings.configurations(serviceName(), "hadoop.conf");
+        Map<String, Object> hadoopConf = LocalSettings.configurations(getServiceName(), "hadoop.conf");
         return (String) hadoopConf.get("content");
     }
 
     public String workers() {
-        Map<String, Object> hdfsConf = LocalSettings.configurations(serviceName(), "workers");
+        Map<String, Object> hdfsConf = LocalSettings.configurations(getServiceName(), "workers");
         return (String) hdfsConf.get("content");
     }
 
     @GlobalParams
     public Map<String, Object> hdfsLog4j() {
-        return LocalSettings.configurations(serviceName(), "hdfs-log4j");
+        return LocalSettings.configurations(getServiceName(), "hdfs-log4j");
     }
 
     @GlobalParams
     public Map<String, Object> coreSite() {
-        Map<String, Object> coreSite = LocalSettings.configurations(serviceName(), "core-site");
+        Map<String, Object> coreSite = LocalSettings.configurations(getServiceName(), "core-site");
         List<String> namenodeList = LocalSettings.hosts("namenode");
         if (!namenodeList.isEmpty()) {
             coreSite.put(
@@ -107,12 +107,12 @@ public Map<String, Object> coreSite() {
 
     @GlobalParams
     public Map<String, Object> hadoopPolicy() {
-        return LocalSettings.configurations(serviceName(), "hadoop-policy");
+        return LocalSettings.configurations(getServiceName(), "hadoop-policy");
     }
 
     @GlobalParams
     public Map<String, Object> hdfsSite() {
-        Map<String, Object> hdfsSite = LocalSettings.configurations(serviceName(), "hdfs-site");
+        Map<String, Object> hdfsSite = LocalSettings.configurations(getServiceName(), "hdfs-site");
         List<String> namenodeList = LocalSettings.hosts("namenode");
         if (!namenodeList.isEmpty()) {
             hdfsSite.put(
@@ -142,12 +142,12 @@ public Map<String, Object> hdfsSite() {
 
     @GlobalParams
     public Map<String, Object> yarnLog4j() {
-        return LocalSettings.configurations(serviceName(), "yarn-log4j");
+        return LocalSettings.configurations(getServiceName(), "yarn-log4j");
     }
 
     @GlobalParams
     public Map<String, Object> yarnSite() {
-        Map<String, Object> yarnSite = LocalSettings.configurations(serviceName(), "yarn-site");
+        Map<String, Object> yarnSite = LocalSettings.configurations(getServiceName(), "yarn-site");
         List<String> resourcemanagerList = LocalSettings.hosts("resourcemanager");
         if (!resourcemanagerList.isEmpty()) {
             yarnSite.put("yarn.resourcemanager.hostname", MessageFormat.format("{0}", resourcemanagerList.get(0)));
@@ -160,22 +160,25 @@ public Map<String, Object> yarnSite() {
 
     @GlobalParams
     public Map<String, Object> mapredSite() {
-        return LocalSettings.configurations(serviceName(), "mapred-site");
+        return LocalSettings.configurations(getServiceName(), "mapred-site");
     }
 
     @GlobalParams
     public Map<String, Object> hadoopEnv() {
-        return LocalSettings.configurations(serviceName(), "hadoop-env");
+        Map<String, Object> configurations = LocalSettings.configurations(getServiceName(), "hadoop-env");
+        configurations.put("hadoop_log_dir", hadoopLogDir);
+        configurations.put("hadoop_pid_dir", hadoopPidDir);
+        return configurations;
     }
 
     @GlobalParams
     public Map<String, Object> yarnEnv() {
-        return LocalSettings.configurations(serviceName(), "yarn-env");
+        return LocalSettings.configurations(getServiceName(), "yarn-env");
     }
 
     @GlobalParams
     public Map<String, Object> mapredEnv() {
-        return LocalSettings.configurations(serviceName(), "mapred-env");
+        return LocalSettings.configurations(getServiceName(), "mapred-env");
     }
 
     @Override
@@ -186,4 +189,9 @@ public String confDir() {
     public String binDir() {
         return serviceHome() + "/bin";
     }
+
+    @Override
+    public String getServiceName() {
+        return "hadoop";
+    }
 }
diff --git a/...igtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HadoopSetup.java b/...igtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HadoopSetup.java
@@ -207,13 +207,13 @@ public static ShellResult configure(Params params, String componentName) {
                 hadoopParams.mapredSite(),
                 hadoopParams.getGlobalParamsMap());
 
-        HdfsUtil.createDirectory(hadoopUser, "/apps");
-        HdfsUtil.createDirectory(hadoopUser, "/app-logs");
-        HdfsUtil.createDirectory(hadoopUser, "/apps/mapred");
-        HdfsUtil.createDirectory(hadoopUser, "/apps/mapred/staging");
-        HdfsUtil.createDirectory(hadoopUser, "/apps/mapred/history");
-        HdfsUtil.createDirectory(hadoopUser, "/apps/mapred/history/tmp");
-        HdfsUtil.createDirectory(hadoopUser, "/apps/mapred/history/done");
+//        HdfsUtil.createDirectory(hadoopUser, "/apps");
+//        HdfsUtil.createDirectory(hadoopUser, "/app-logs");
+//        HdfsUtil.createDirectory(hadoopUser, "/apps/mapred");
+//        HdfsUtil.createDirectory(hadoopUser, "/apps/mapred/staging");
+//        HdfsUtil.createDirectory(hadoopUser, "/apps/mapred/history");
+//        HdfsUtil.createDirectory(hadoopUser, "/apps/mapred/history/tmp");
+//        HdfsUtil.createDirectory(hadoopUser, "/apps/mapred/history/done");
 
         log.info("Successfully configured Hadoop");
         return ShellResult.success();

diff --git a/...bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaParams.java b/...bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaParams.java
@@ -54,14 +54,14 @@ public KafkaParams(CommandPayload commandPayload) {
 
     @GlobalParams
     public Map<String, Object> kafkaBroker() {
-        Map<String, Object> kafkaBroker = LocalSettings.configurations(serviceName(), "kafka-broker");
+        Map<String, Object> kafkaBroker = LocalSettings.configurations(getServiceName(), "kafka-broker");
         kafkaDataDir = (String) kafkaBroker.get("log.dirs");
         return kafkaBroker;
     }
 
     @GlobalParams
     public Map<String, Object> kafkaEnv() {
-        Map<String, Object> kafkaEnv = LocalSettings.configurations(serviceName(), "kafka-env");
+        Map<String, Object> kafkaEnv = LocalSettings.configurations(getServiceName(), "kafka-env");
         kafkaPidDir = (String) kafkaEnv.get("kafka_pid_dir");
         kafkaPidFile = kafkaPidDir + "/kafka_broker.pid";
         kafkaLogDir = (String) kafkaEnv.get("kafka_log_dir");
@@ -71,13 +71,18 @@ public Map<String, Object> kafkaEnv() {
 
     @GlobalParams
     public Map<String, Object> kafkaLog4j() {
-        Map<String, Object> kafkaLog4j = LocalSettings.configurations(serviceName(), "kafka-log4j");
+        Map<String, Object> kafkaLog4j = LocalSettings.configurations(getServiceName(), "kafka-log4j");
         kafkaLog4jContent = (String) kafkaLog4j.get("content");
         return kafkaLog4j;
     }
 
     public String kafkaLimits() {
-        Map<String, Object> kafkaLimits = LocalSettings.configurations(serviceName(), "kafka.conf");
+        Map<String, Object> kafkaLimits = LocalSettings.configurations(getServiceName(), "kafka.conf");
         return (String) kafkaLimits.get("content");
     }
+
+    @Override
+    public String getServiceName() {
+        return "kafka";
+    }
 }
diff --git a/...k-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrParams.java b/...k-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrParams.java
@@ -59,16 +59,16 @@ public SolrParams(CommandPayload commandPayload) {
 
     @GlobalParams
     public Map<String, Object> solrXml() {
-        return LocalSettings.configurations(serviceName(), "solr-xml");
+        return LocalSettings.configurations(getServiceName(), "solr-xml");
     }
 
     @GlobalParams
     public Map<String, Object> solrLog4j() {
-        return LocalSettings.configurations(serviceName(), "solr-log4j");
+        return LocalSettings.configurations(getServiceName(), "solr-log4j");
     }
 
     public String getZnode() {
-        Map<String, Object> solrEnv = LocalSettings.configurations(serviceName(), "solr-env");
+        Map<String, Object> solrEnv = LocalSettings.configurations(getServiceName(), "solr-env");
         return (String) solrEnv.get("solr_znode");
     }
 
@@ -87,12 +87,17 @@ public String zkHost() {
 
     @GlobalParams
     public Map<String, Object> solrEnv() {
-        Map<String, Object> solrEnv = LocalSettings.configurations(serviceName(), "solr-env");
+        Map<String, Object> solrEnv = LocalSettings.configurations(getServiceName(), "solr-env");
         solrLogDir = (String) solrEnv.get("solr_log_dir");
         solrPidDir = (String) solrEnv.get("solr_pid_dir");
         solrDataDir = (String) solrEnv.get("solr_datadir");
         solrPort = (String) solrEnv.get("SOLR_PORT");
         solrPidFile = solrPidDir + "/solr-" + solrPort + ".pid";
         return solrEnv;
     }
+
+    @Override
+    public String getServiceName() {
+        return "solr";
+    }
 }