优化镜像

Dockerfile

@@ -9,7 +9,6 @@ RUN yum makecache \
     && wget -O /tmp/sqoop.bin.tar.gz https://archive.apache.org/dist/sqoop/1.4.7/sqoop-1.4.7.bin__hadoop-2.6.0.tar.gz \
     && wget -O /tmp/spark.bin.tar.gz https://dlcdn.apache.org/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3-scala2.13.tgz \
     && wget -O /tmp/mysql-connector-j.tar.gz https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-j-8.1.0.tar.gz \
-    && wget -O /tmp/mysql.rpm https://dev.mysql.com/get/mysql80-community-release-el7-10.noarch.rpm \
     && wget -O /tmp/s6-overlay-noarch.tar.xz https://github.com/just-containers/s6-overlay/releases/download/v3.1.5.0/s6-overlay-noarch.tar.xz \
     && wget -O /tmp/scala.tar.gz https://downloads.lightbend.com/scala/2.13.12/scala-2.13.12.tgz \
     && if [ "$(uname -m)" = "aarch64" ]; then \
@@ -57,18 +56,18 @@ RUN tar -zxvf /tmp/jdk.tar.gz -C /usr/local/software/ > /dev/null \
     && mv /tmp/ssh /usr/local/software/data/ \
     && mv /tmp/s6-overlay-noarch.tar.xz /usr/local/software/data/ \
     && mv /tmp/s6-overlay.tar.xz /usr/local/software/data/ \
-    && mv /tmp/s6-rc.d /usr/local/software/data/ \
-    && mv /tmp/mysql.rpm /usr/local/software/data/
+    && mv /tmp/s6-rc.d /usr/local/software/data/
 
 FROM centos:centos7 as runtime
-COPY --from=builder /usr/local/software /usr/local/software
-
-RUN rpm -ivh /usr/local/software/data/mysql.rpm \
+ADD https://dev.mysql.com/get/mysql80-community-release-el7-10.noarch.rpm /tmp/mysql.rpm
+RUN rpm -ivh /tmp/mysql.rpm \
     && yum update -y \
     && yum install -y rsync wget tar which net-tools openssh-clients openssh-server passwd openssl kde-l10n-Chinese sudo mysql-community-client nc \
     && yum clean all \
-    && rm -rf /var/cache/yum \
-    && mkdir -p /root/.ssh \
+    && rm -rf /tmp/*
+
+COPY --from=builder /usr/local/software /usr/local/software
+RUN mkdir -p /root/.ssh \
     && mv /usr/local/software/data/bin /root/ \
     && mv /usr/local/software/data/ssh_key/* /etc/ssh/ \
     && mv /usr/local/software/data/ssh/* /root/.ssh/ \

bin/s6-rc.d/hadoop/run

@@ -19,27 +19,25 @@ if [[ "$HOSTNAME" == *"master"* ]]; then
     /usr/local/software/hadoop-3.3.6/bin/hadoop fs -mkdir -p /spark/spark-log
     /usr/local/software/spark-3.5.0/sbin/start-all.sh
     /usr/local/software/spark-3.5.0/sbin/start-history-server.sh
-    /root/bin/zk start
+    /root/bin/zk restart
     echo "Y" | yarn rmadmin -transitionToStandby --forcemanual rm2 > /dev/null 2>&1
     echo "Y" | yarn rmadmin -transitionToActive --forcemanual rm1 > /dev/null 2>&1
     # 验证数据库是否存在
-    if mysql -u root -proot -h mysql -e "use hive" > /dev/null 2>&1; then
+    if mysql -uroot -proot -hmysql -e"use hive" > /dev/null 2>&1; then
         echo "数据库已存在，跳过初始化"
     else
         /usr/local/software/hive-3.1.3/bin/schematool -initSchema -dbType mysql > /dev/null 2>&1
         echo "数据库初始化完成"
     fi
-    nohup /usr/local/software/hive-3.1.3/bin/hive --service metastore 2>&1 &
     echo "启动hive元数据服务"
-    echo "等待启动HiveServer2服务"
-    echo "hive元数据服务未就绪，等待中..."
+    nohup /usr/local/software/hive-3.1.3/bin/hive --service metastore 2>&1 &
     while true; do
         # 检查端口是否开放
         if netstat -an | grep 9083 > /dev/null; then
             # 启动HiveServer2服务
-            /usr/local/software/hadoop-3.3.6/bin/hdfs dfsadmin -safemode leave
-            nohup /usr/local/software/hive-3.1.3/bin/hive --service hiveserver2 2>&1 &
+            echo "hive元数据服务启动完成"
             echo "启动HiveServer2服务"
+            nohup /usr/local/software/hive-3.1.3/bin/hive --service hiveserver2 2>&1 &
             break
         else
             sleep 1

docker-compose.yml

@@ -9,9 +9,6 @@ services:
       - "8088:8088"
       - "9000:22"
       - "8080:8080"
-    networks:
-      hadoop:
-        ipv4_address: 172.28.1.1
     depends_on:
       - slave1
       - slave2
@@ -20,16 +17,10 @@ services:
     image: pkc278/hadoop:latest
     container_name: CentOS_slave1
     hostname: slave1
-    networks:
-      hadoop:
-        ipv4_address: 172.28.1.2
   slave2:
     image: pkc278/hadoop:latest
     container_name: CentOS_slave2
     hostname: slave2
-    networks:
-      hadoop:
-        ipv4_address: 172.28.1.3
   mysql:
     image: mysql:latest
     container_name: MySQL
@@ -38,13 +29,3 @@ services:
       - MYSQL_ROOT_PASSWORD=root
     ports:
       - "3316:3306"
-    networks:
-      hadoop:
-        ipv4_address: 172.28.1.4
-networks:
-  hadoop:
-    driver: bridge
-    ipam:
-      driver: default
-      config:
-        - subnet: 172.28.0.0/16