revert spark version to 3.1.3 (#9385)

* revert spark version to 3.1.3 * update
intel-analytics · Nov 9, 2023 · fc3e1a7 · fc3e1a7
1 parent 382745d
commit fc3e1a7
Show file tree

Hide file tree

Showing 31 changed files with 136 additions and 202 deletions.
diff --git a/.github/actions/dllib-scala-ut-action/action.yml b/.github/actions/dllib-scala-ut-action/action.yml
@@ -9,14 +9,14 @@ runs:
         export SPARK_LOCAL_HOSTNAME=localhost
         export KERAS_BACKEND=tensorflow
         cd scala
-        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.optim" "-Dtest=com.intel.analytics.bigdl.dllib.optim.*Test" test -P spark_3.x -Dspark.version=3.4.1 
-        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.models" "-Dtest=com.intel.analytics.bigdl.dllib.models.*Test" test -P spark_3.x -Dspark.version=3.4.1 
-        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.estimator" "-Dtest=com.intel.analytics.bigdl.dllib.estimator.*Test" test -P spark_3.x -Dspark.version=3.4.1
-        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.nnframes" "-Dtest=com.intel.analytics.bigdl.dllib.nnframes.*Test" test -P spark_3.x -Dspark.version=3.4.1
-        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.feature" "-Dtest=com.intel.analytics.bigdl.dllib.feature.*Test" test -P spark_3.x -Dspark.version=3.4.1
-        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.intermediate" "-Dtest=com.intel.analytics.bigdl.dllib.utils.intermediate.*Test" test -P spark_3.x -Dspark.version=3.4.1
-        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.tf" "-Dtest=com.intel.analytics.bigdl.dllib.utils.tf.*Test" test -P spark_3.x -Dspark.version=3.4.1
-        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.python.api" "-Dtest=com.intel.analytics.bigdl.dllib.utils.python.api.*Test" test -P spark_3.x -Dspark.version=3.4.1
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.optim" "-Dtest=com.intel.analytics.bigdl.dllib.optim.*Test" test -P spark_3.x -Dspark.version=3.1.3 
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.models" "-Dtest=com.intel.analytics.bigdl.dllib.models.*Test" test -P spark_3.x -Dspark.version=3.1.3 
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.estimator" "-Dtest=com.intel.analytics.bigdl.dllib.estimator.*Test" test -P spark_3.x -Dspark.version=3.1.3
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.nnframes" "-Dtest=com.intel.analytics.bigdl.dllib.nnframes.*Test" test -P spark_3.x -Dspark.version=3.1.3
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.feature" "-Dtest=com.intel.analytics.bigdl.dllib.feature.*Test" test -P spark_3.x -Dspark.version=3.1.3
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.intermediate" "-Dtest=com.intel.analytics.bigdl.dllib.utils.intermediate.*Test" test -P spark_3.x -Dspark.version=3.1.3
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.tf" "-Dtest=com.intel.analytics.bigdl.dllib.utils.tf.*Test" test -P spark_3.x -Dspark.version=3.1.3
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.python.api" "-Dtest=com.intel.analytics.bigdl.dllib.utils.python.api.*Test" test -P spark_3.x -Dspark.version=3.1.3
         #need python requirements
         #mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.keras" "-Dtest=com.intel.analytics.bigdl.dllib.keras.*Test" test -P spark_3.x -Dspark.version=3.1.3 
         #mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.nn.mkldnn" "-Dtest=com.intel.analytics.bigdl.dllib.nn.mkldnn.*Test" test -P spark_3.x -Dspark.version=3.1.3 
diff --git a/.github/actions/friesian-scala-ut-action/action.yml b/.github/actions/friesian-scala-ut-action/action.yml
@@ -39,4 +39,4 @@ runs:
         export KERAS_BACKEND=tensorflow
         cd scala
         mvn "-DwildcardSuites=com.intel.analytics.bigdl.friesian" "-Dtest=com.intel.analytics.bigdl.friesian.*Test" test -P serving,spark_2.x -Dspark.version=2.4.6 "-Dspark.master=local[*]" -pl "!ppml"
-        mvn "-DwildcardSuites=com.intel.analytics.bigdl.friesian" "-Dtest=com.intel.analytics.bigdl.friesian.*Test" test -P serving,spark_3.x -Dspark.version=3.4.1 "-Dspark.master=local[*]"
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.friesian" "-Dtest=com.intel.analytics.bigdl.friesian.*Test" test -P serving,spark_3.x -Dspark.version=3.1.3 "-Dspark.master=local[*]"
diff --git a/.github/actions/friesian/friesian-python-exampletest-feature-tf1-py37-spark3/action.yml b/.github/actions/friesian/friesian-python-exampletest-feature-tf1-py37-spark3/action.yml
@@ -7,17 +7,17 @@ runs:
       shell: bash
       run: |
         cd scala
-        ./make-dist.sh -P spark_3.x -Dspark.version=3.4.1
+        ./make-dist.sh -P spark_3.x -Dspark.version=3.1.3
         cd -
        
-        wget https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3.tgz -O ${{ github.workspace }}/spark-3.4.1-bin-hadoop3.tgz
-        tar -xf spark-3.4.1-bin-hadoop3.tgz -C ${{ github.workspace }}/
-        export SPARK_HOME=${{ github.workspace }}/spark-3.4.1-bin-hadoop3
+        wget https://archive.apache.org/dist/spark/spark-3.1.3/spark-3.1.3-bin-hadoop2.7.tgz -O ${{ github.workspace }}/spark-3.1.3-bin-hadoop2.7.tgz
+        tar -xf spark-3.1.3-bin-hadoop2.7.tgz -C ${{ github.workspace }}/
+        export SPARK_HOME=${{ github.workspace }}/spark-3.1.3-bin-hadoop2.7
         
         source activate py37
         source python/friesian/dev/prepare_env.sh
         export SPARK_LOCAL_HOSTNAME=localhost
-        pip install pyspark==3.4.1 
+        pip install pyspark==3.1.3 
         pip install -i https://pypi.org/simple bigdl-math bigdl-tf
         pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} grpcio==1.43.0
 

diff --git a/.github/actions/friesian/friesian-python-exampletest-tf2-py37-spark3/action.yml b/.github/actions/friesian/friesian-python-exampletest-tf2-py37-spark3/action.yml
@@ -7,17 +7,17 @@ runs:
       shell: bash
       run: |
         cd scala
-        ./make-dist.sh -P spark_3.x -Dspark.version=3.4.1
+        ./make-dist.sh -P spark_3.x -Dspark.version=3.1.3
         cd -
        
-        wget https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3.tgz -O ${{ github.workspace }}/spark-3.4.1-bin-hadoop3.tgz
-        tar -xf spark-3.4.1-bin-hadoop3.tgz -C ${{ github.workspace }}/
-        export SPARK_HOME=${{ github.workspace }}/spark-3.4.1-bin-hadoop3
+        wget https://archive.apache.org/dist/spark/spark-3.1.3/spark-3.1.3-bin-hadoop2.7.tgz -O ${{ github.workspace }}/spark-3.1.3-bin-hadoop2.7.tgz
+        tar -xf spark-3.1.3-bin-hadoop2.7.tgz -C ${{ github.workspace }}/
+        export SPARK_HOME=${{ github.workspace }}/spark-3.1.3-bin-hadoop2.7
         
         source activate py37
         source python/friesian/dev/prepare_env.sh
         export SPARK_LOCAL_HOSTNAME=localhost
-        pip install pyspark==3.4.1 
+        pip install pyspark==3.1.3 
         pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} grpcio==1.43.0
 
         cd python/friesian/dev/test

diff --git a/.github/actions/friesian/friesian-python-py37-spark3-pip/action.yml b/.github/actions/friesian/friesian-python-py37-spark3-pip/action.yml
@@ -7,7 +7,7 @@ runs:
       shell: bash
       run: |
         cd scala
-        mvn clean package -DskipTests -P spark_3.x -Dspark.version=3.4.1 -U
+        mvn clean package -DskipTests -P spark_3.x -Dspark.version=3.1.3 -U
         cd -
        
         #wget https://archive.apache.org/dist/spark/spark-3.1.3/spark-3.1.3-bin-hadoop2.7.tgz -O ${{ github.workspace }}/spark-3.1.3-bin-hadoop2.7.tgz
@@ -16,7 +16,7 @@ runs:
       
         source activate py37
         export SPARK_LOCAL_HOSTNAME=localhost
-        pip install pyspark==3.4.1 
+        pip install pyspark==3.1.3 
         chmod a+x python/dev/release_default_linux_spark3.sh
         ./python/dev/release_default_linux_spark3.sh default false false
         

diff --git a/.github/actions/friesian/friesian-python-py37-spark3/action.yml b/.github/actions/friesian/friesian-python-py37-spark3/action.yml
@@ -7,16 +7,16 @@ runs:
       shell: bash
       run: |
         cd scala
-        ./make-dist.sh -P spark_3.x -Dspark.version=3.4.1
+        ./make-dist.sh -P spark_3.x -Dspark.version=3.1.3
         cd -
        
-        wget https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3.tgz -O ${{ github.workspace }}/spark-3.4.1-bin-hadoop3.tgz
-        tar -xf spark-3.4.1-bin-hadoop3.tgz -C ${{ github.workspace }}/
-        export SPARK_HOME=${{ github.workspace }}/spark-3.4.1-bin-hadoop3
+        wget https://archive.apache.org/dist/spark/spark-3.1.3/spark-3.1.3-bin-hadoop2.7.tgz -O ${{ github.workspace }}/spark-3.1.3-bin-hadoop2.7.tgz
+        tar -xf spark-3.1.3-bin-hadoop2.7.tgz -C ${{ github.workspace }}/
+        export SPARK_HOME=${{ github.workspace }}/spark-3.1.3-bin-hadoop2.7
 
         source activate py37
         export SPARK_LOCAL_HOSTNAME=localhost
-        pip install pyspark==3.4.1 
+        pip install pyspark==3.1.3 
         chmod a+x python/friesian/dev/test/run-pytests-friesian
         python/friesian/dev/test/run-pytests-friesian
         source deactivate

diff --git a/.github/actions/mac/dllib/mac-dllib-scalatest-spark3/action.yml b/.github/actions/mac/dllib/mac-dllib-scalatest-spark3/action.yml
@@ -1,5 +1,5 @@
-name: 'Mac Dllib ScalaTest Spark3.4'
-description: 'Mac Dllib ScalaTest Spark3.4'
+name: 'Mac Dllib ScalaTest Spark3.1'
+description: 'Mac Dllib ScalaTest Spark3.1'
 runs:
   using: "composite"
   steps:
@@ -8,13 +8,13 @@ runs:
       run: |
         source ~/.bash_profile
         source activate py37
-        export SPARK_HOME=/opt/work/spark-3.4.1
+        export SPARK_HOME=/opt/work/spark-3.1.3
         export KERAS_BACKEND=tensorflow
         # rm /var/root/.m2/repository/io/netty/netty-common/4.1.50.Final/netty-common-4.1.50.Final.jar
         cd scala
-        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.estimator" test -P spark_3.x -Dspark.version=3.4.1 -pl '!orca,!friesian,!grpc,!serving,!ppml,!assembly' && \
-        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.nnframes" test -P spark_3.x -Dspark.version=3.4.1 -pl '!orca,!friesian,!grpc,!serving,!ppml,!assembly' && \
-        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.keras" test -P spark_3.x -Dspark.version=3.4.1 -pl '!orca,!friesian,!grpc,!serving,!ppml,!assembly' 
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.estimator" test -P spark_3.x -Dspark.version=3.1.3 -pl '!orca,!friesian,!grpc,!serving,!ppml,!assembly' && \
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.nnframes" test -P spark_3.x -Dspark.version=3.1.3 -pl '!orca,!friesian,!grpc,!serving,!ppml,!assembly' && \
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.keras" test -P spark_3.x -Dspark.version=3.1.3 -pl '!orca,!friesian,!grpc,!serving,!ppml,!assembly' 
         cd -
         conda deactivate
       env:

diff --git a/...ub/actions/mac/friesian/mac-friesian-python-exampletest-feature-tf1-pip-spark3/action.yml b/...ub/actions/mac/friesian/mac-friesian-python-exampletest-feature-tf1-pip-spark3/action.yml
@@ -10,23 +10,23 @@ runs:
         source activate py37
         pip uninstall -y bigdl-dllib bigdl-orca bigdl-chronos bigdl-orca-spark3 bigdl-dllib-spark3 bigdl-chronos-spark3  pyspark
         export KERAS_BACKEND=tensorflow
-        export SPARK_HOME=/opt/work/spark-3.4.1
+        export SPARK_HOME=/opt/work/spark-3.1.3
         PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH"
         
-        gsed -i "s/'pyspark=='+PYSPARK_VERSION/'pyspark==3.4.1'/g" python/dllib/src/setup.py
+        gsed -i "s/'pyspark=='+PYSPARK_VERSION/'pyspark==3.1.3'/g" python/dllib/src/setup.py
         gsed -i "s/name='bigdl-dllib'/name='bigdl-dllib-spark3'/g" python/dllib/src/setup.py
         gsed -i "s/dist\/bigdl_dllib-/dist\/bigdl_dllib_spark3-/g" python/dllib/dev/release/release.sh
-        bash python/dllib/dev/release/release.sh mac default false false -Dspark.version=3.4.1 -P spark_3.x ${profiles}
+        bash python/dllib/dev/release/release.sh mac default false false -Dspark.version=3.1.3 -P spark_3.x ${profiles}
 
         gsed -i "s/bigdl-dllib==/bigdl-dllib-spark3==/g" python/orca/src/setup.py
         gsed -i "s/name='bigdl-orca'/name='bigdl-orca-spark3'/g" python/orca/src/setup.py
         gsed -i "s/dist\/bigdl_orca-/dist\/bigdl_orca_spark3-/g" python/orca/dev/release/release.sh
-        bash python/orca/dev/release/release.sh mac default true false -Dspark.version=3.4.1 -P spark_3.x
+        bash python/orca/dev/release/release.sh mac default true false -Dspark.version=3.1.3 -P spark_3.x
 
         gsed -i "s/bigdl-orca==/bigdl-orca-spark3==/g" python/friesian/src/setup.py
         gsed -i "s/name='bigdl-friesian'/name='bigdl-friesian-spark3'/g" python/friesian/src/setup.py
         gsed -i "s/dist\/bigdl_friesian-/dist\/bigdl_friesian_spark3-/g" python/friesian/dev/release/release.sh
-        bash python/friesian/dev/release/release.sh mac default false false -Dspark.version=3.4.1 -P spark_3.x
+        bash python/friesian/dev/release/release.sh mac default false false -Dspark.version=3.1.3 -P spark_3.x
        
         echo "Running py37 tests"
         # install dllib

diff --git a/.github/actions/mac/friesian/mac-friesian-python-exampletest-feature-tf1-spark3/action.yml b/.github/actions/mac/friesian/mac-friesian-python-exampletest-feature-tf1-spark3/action.yml
@@ -7,16 +7,16 @@ runs:
       shell: bash
       run: |
         source ~/.bash_profile
-        export SPARK_HOME=/opt/work/spark-3.4.1
+        export SPARK_HOME=/opt/work/spark-3.1.3
         PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH"
         cd scala
-        ./make-dist.sh -P spark_3.x -Dspark.version=3.4.1
+        ./make-dist.sh -P spark_3.x -Dspark.version=3.1.3
         cd -
         source python/friesian/dev/prepare_env.sh
         source activate py37
         pip uninstall -y bigdl-dllib bigdl-orca bigdl-chronos bigdl-orca-spark3 bigdl-dllib-spark3 bigdl-chronos-spark3  pyspark
         
-        pip install pyspark==3.4.1
+        pip install pyspark==3.1.3
         pip install -i https://pypi.python.org/simple bigdl-tf bigdl-math
         echo "Running py37 tests"
         cd python/friesian/dev/test

diff --git a/.github/actions/mac/friesian/mac-friesian-python-exampletest-tf2-pip-spark3/action.yml b/.github/actions/mac/friesian/mac-friesian-python-exampletest-tf2-pip-spark3/action.yml
@@ -10,22 +10,22 @@ runs:
         source activate py37-tf2
         pip uninstall -y bigdl-dllib bigdl-orca bigdl-chronos bigdl-orca-spark3 bigdl-dllib-spark3 bigdl-chronos-spark3 bigdl-friesian-spark3  pyspark
         export KERAS_BACKEND=tensorflow
-        export SPARK_HOME=/opt/work/spark-3.4.1
+        export SPARK_HOME=/opt/work/spark-3.1.3
         PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH"
-        gsed -i "s/'pyspark=='+PYSPARK_VERSION/'pyspark==3.4.1'/g" python/dllib/src/setup.py
+        gsed -i "s/'pyspark=='+PYSPARK_VERSION/'pyspark==3.1.3'/g" python/dllib/src/setup.py
         gsed -i "s/name='bigdl-dllib'/name='bigdl-dllib-spark3'/g" python/dllib/src/setup.py
         gsed -i "s/dist\/bigdl_dllib-/dist\/bigdl_dllib_spark3-/g" python/dllib/dev/release/release.sh
-        bash python/dllib/dev/release/release.sh mac default false false -Dspark.version=3.4.1 -P spark_3.x ${profiles}
+        bash python/dllib/dev/release/release.sh mac default false false -Dspark.version=3.1.3 -P spark_3.x ${profiles}
 
         gsed -i "s/bigdl-dllib==/bigdl-dllib-spark3==/g" python/orca/src/setup.py
         gsed -i "s/name='bigdl-orca'/name='bigdl-orca-spark3'/g" python/orca/src/setup.py
         gsed -i "s/dist\/bigdl_orca-/dist\/bigdl_orca_spark3-/g" python/orca/dev/release/release.sh
-        bash python/orca/dev/release/release.sh mac default true false -Dspark.version=3.4.1 -P spark_3.x
+        bash python/orca/dev/release/release.sh mac default true false -Dspark.version=3.1.3 -P spark_3.x
 
         gsed -i "s/bigdl-orca==/bigdl-orca-spark3==/g" python/friesian/src/setup.py
         gsed -i "s/name='bigdl-friesian'/name='bigdl-friesian-spark3'/g" python/friesian/src/setup.py
         gsed -i "s/dist\/bigdl_friesian-/dist\/bigdl_friesian_spark3-/g" python/friesian/dev/release/release.sh
-        bash python/friesian/dev/release/release.sh mac default false false -Dspark.version=3.4.1 -P spark_3.x
+        bash python/friesian/dev/release/release.sh mac default false false -Dspark.version=3.1.3 -P spark_3.x
        
         echo "Running py37 tests"
         # install dllib

diff --git a/.github/actions/mac/friesian/mac-friesian-python-exampletest-tf2-spark3/action.yml b/.github/actions/mac/friesian/mac-friesian-python-exampletest-tf2-spark3/action.yml
@@ -7,16 +7,16 @@ runs:
       shell: bash
       run: |
         source ~/.bash_profile
-        export SPARK_HOME=/opt/work/spark-3.4.1
+        export SPARK_HOME=/opt/work/spark-3.1.3
         PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH"
         cd scala
-        ./make-dist.sh -P spark_3.x -Dspark.version=3.4.1
+        ./make-dist.sh -P spark_3.x -Dspark.version=3.1.3
         cd -
         source python/friesian/dev/prepare_env.sh
         source activate py37-tf2
         pip uninstall -y bigdl-dllib bigdl-orca bigdl-chronos bigdl-orca-spark3 bigdl-dllib-spark3 bigdl-chronos-spark3  pyspark
         
-        pip install pyspark==3.4.1
+        pip install pyspark==3.1.3
         echo "Running py37 tests"
         cd python/friesian/dev/test
         gsed -i "s/sed/gsed/g" run-example-tests-tf2.6.0.sh

diff --git a/.github/actions/mac/friesian/mac-friesian-python-ut-pip-spark3/action.yml b/.github/actions/mac/friesian/mac-friesian-python-ut-pip-spark3/action.yml
@@ -10,22 +10,22 @@ runs:
         source activate py37
         pip uninstall -y bigdl-dllib bigdl-orca bigdl-chronos bigdl-orca-spark3 bigdl-dllib-spark3 bigdl-chronos-spark3  pyspark
         export KERAS_BACKEND=tensorflow
-        export SPARK_HOME=/opt/work/spark-3.4.1
+        export SPARK_HOME=/opt/work/spark-3.1.3
 
-        gsed -i "s/'pyspark=='+PYSPARK_VERSION/'pyspark==3.4.1'/g" python/dllib/src/setup.py
+        gsed -i "s/'pyspark=='+PYSPARK_VERSION/'pyspark==3.1.3'/g" python/dllib/src/setup.py
         gsed -i "s/name='bigdl-dllib'/name='bigdl-dllib-spark3'/g" python/dllib/src/setup.py
         gsed -i "s/dist\/bigdl_dllib-/dist\/bigdl_dllib_spark3-/g" python/dllib/dev/release/release.sh
-        bash python/dllib/dev/release/release.sh mac default false false -Dspark.version=3.4.1 -P spark_3.x ${profiles}
+        bash python/dllib/dev/release/release.sh mac default false false -Dspark.version=3.1.3 -P spark_3.x ${profiles}
 
         gsed -i "s/bigdl-dllib==/bigdl-dllib-spark3==/g" python/orca/src/setup.py
         gsed -i "s/name='bigdl-orca'/name='bigdl-orca-spark3'/g" python/orca/src/setup.py
         gsed -i "s/dist\/bigdl_orca-/dist\/bigdl_orca_spark3-/g" python/orca/dev/release/release.sh
-        bash python/orca/dev/release/release.sh mac default true false -Dspark.version=3.4.1 -P spark_3.x
+        bash python/orca/dev/release/release.sh mac default true false -Dspark.version=3.1.3 -P spark_3.x
 
         gsed -i "s/bigdl-orca==/bigdl-orca-spark3==/g" python/friesian/src/setup.py
         gsed -i "s/name='bigdl-friesian'/name='bigdl-friesian-spark3'/g" python/friesian/src/setup.py
         gsed -i "s/dist\/bigdl_friesian-/dist\/bigdl_friesian_spark3-/g" python/friesian/dev/release/release.sh
-        bash python/friesian/dev/release/release.sh mac default false false -Dspark.version=3.4.1 -P spark_3.x
+        bash python/friesian/dev/release/release.sh mac default false false -Dspark.version=3.1.3 -P spark_3.x
        
         echo "Running py37 tests"
         # install dllib

diff --git a/.github/actions/mac/friesian/mac-friesian-python-ut-spark3/action.yml b/.github/actions/mac/friesian/mac-friesian-python-ut-spark3/action.yml
@@ -10,11 +10,11 @@ runs:
         source activate py37
         pip uninstall -y bigdl-dllib bigdl-orca bigdl-chronos bigdl-orca-spark3 bigdl-dllib-spark3 bigdl-chronos-spark3  pyspark
         export KERAS_BACKEND=tensorflow
-        export SPARK_HOME=/opt/work/spark-3.4.1
+        export SPARK_HOME=/opt/work/spark-3.1.3
         cd scala
-        ./make-dist.sh -P spark_3.x -Dspark.version=3.4.1
+        ./make-dist.sh -P spark_3.x -Dspark.version=3.1.3
         cd -
-        pip install pyspark==3.4.1
+        pip install pyspark==3.1.3
         echo "Running py37 tests"
         python/friesian/dev/test/run-pytests-friesian
 

diff --git a/.github/actions/mac/orca/mac-orca-python-ut-spark3/action.yml b/.github/actions/mac/orca/mac-orca-python-ut-spark3/action.yml
@@ -10,7 +10,7 @@ runs:
         source activate py37
         pip uninstall -y bigdl-dllib bigdl-orca bigdl-chronos bigdl-orca-spark3 bigdl-dllib-spark3 bigdl-chronos-spark3  pyspark
         export KERAS_BACKEND=tensorflow
-        #export SPARK_HOME=/opt/work/spark-3.4.1
+        #export SPARK_HOME=/opt/work/spark-3.1.3
         export MASTER=local[4]
         export HDFS_URI=/Users/arda/data
         export PATH=/opt/maven/apache-maven-3.6.3/bin:$PATH

diff --git a/.github/workflows/chronos-example-python-spark31.yml b/.github/workflows/chronos-example-python-spark31.yml
@@ -53,7 +53,7 @@ jobs:
           apt-get install -y libgl1
           apt-get install patchelf
           pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-nano bigdl-friesian bigdl-friesian-spark3
-          sed -i "s/pyspark==2.4.6/pyspark==3.4.1/g" python/dllib/src/setup.py
+          sed -i "s/pyspark==2.4.6/pyspark==3.1.3/g" python/dllib/src/setup.py
           wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-chronos-python-ut.txt -O ${{ github.workspace }}/requirements-chronos-python-ut.txt
           pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} -r ${{ github.workspace }}/requirements-chronos-python-ut.txt
           bash python/dev/release_default_linux_spark3.sh default false false