-
Notifications
You must be signed in to change notification settings - Fork 1.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into cicd_release_zip
- Loading branch information
Showing
270 changed files
with
15,123 additions
and
6,140 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
191 changes: 191 additions & 0 deletions
191
.github/actions/ppml/ppml-occlum-EDMM-k8s-exampletests-action/action.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,191 @@ | ||
| name: 'Run PPML Occlum k8s ExampleTests' | ||
| description: 'Run PPML Occlum k8s ExampleTests' | ||
| inputs: | ||
| image: | ||
| description: 'image' | ||
| required: true | ||
| default: '10.239.45.10/arda/intelanalytics/bigdl-ppml-trusted-big-data-ml-scala-occlum' | ||
| image-tag: | ||
| description: 'image tag' | ||
| required: true | ||
| default: '2.4.0-SNAPSHOT-EDMM' | ||
| runs: | ||
| using: "composite" | ||
| steps: | ||
| - name: Run tests | ||
| shell: bash | ||
| env: | ||
| DEFAULT_IMAGE: ${{ inputs.image }}:${{ inputs.image-tag }} | ||
| run: | | ||
| whoami | ||
| export IMAGE=${{ env.DEFAULT_IMAGE }} | ||
| docker pull $IMAGE | ||
| docker pull intelanalytics/bigdl-ppml-trusted-big-data-ml-scala-occlum:2.4.0-SNAPSHOT-EDMM | ||
| export NO_PROXY=10.239.45.10:8081,10.112.231.51,10.239.45.10,172.168.0.205,172.168.0.210 | ||
| export kubernetes_master_url=172.168.0.210 | ||
| export SPARK_HOME=/opt/spark-3.1.2-bin-hadoop3.2 | ||
| cd /home/icx/BigDL/ppml/trusted-big-data-ml/scala/docker-occlum/kubernetes | ||
| status_1_spark_pi=1 | ||
| status_2_spark_lr=1 | ||
| status_3_spark_sql=1 | ||
| status_4_spark_gbt=1 | ||
| status_5_spark_gbt_criteo=1 | ||
| status_6_spark_tpch=1 | ||
| status_7_pyspark_sklearn=1 | ||
| status_8_pyspark_sql=1 | ||
| status_9_pyspark_tpch=1 | ||
| status_10_spark_lgbm=1 | ||
| if [ $status_1_spark_pi -ne 0 ]; then | ||
| echo "################## start spark pi" | ||
| echo "example.1 spark pi" | ||
| bash run_spark_pi.sh $IMAGE | ||
| if [ "`kubectl get pods | grep spark-pi-test | awk '{print $1}'`" != "" ]; then | ||
| pi_driver=`kubectl get pods | grep spark-pi-test | awk '{print $1}' | grep driver` | ||
| fi | ||
| kubectl logs $pi_driver | egrep 'Pi is roughly 3' | ||
| status_1_spark_pi=$(echo $?) | ||
| if [ "`kubectl get pods | grep spark-pi-test | awk '{print $1}'`" != "" ]; then | ||
| kubectl delete pods $pi_driver | ||
| fi | ||
| fi | ||
| if [ $status_2_spark_lr -ne 0 ]; then | ||
| echo "################## start spark lr" | ||
| echo "example.2 spark lr" | ||
| bash run_spark_lr.sh $IMAGE | ||
| if [ "`kubectl get pods | grep spark-lr-test | awk '{print $1}'`" != "" ]; then | ||
| lr_driver=`kubectl get pods | grep spark-lr-test | awk '{print $1}' | grep driver` | ||
| fi | ||
| kubectl logs $lr_driver | egrep 'Training data results' | ||
| status_2_spark_lr=$(echo $?) | ||
| if [ "`kubectl get pods | grep spark-lr-test | awk '{print $1}'`" != "" ]; then | ||
| kubectl delete pods $lr_driver | ||
| fi | ||
| fi | ||
| if [ $status_3_spark_sql -ne 0 ]; then | ||
| echo "################## start spark sql" | ||
| echo "example.3 spark sql" | ||
| bash run_spark_sql.sh $IMAGE | ||
| if [ "`kubectl get pods | grep spark-sql-test | awk '{print $1}'`" != "" ]; then | ||
| sql_driver=`kubectl get pods | grep spark-sql-test | awk '{print $1}' | grep driver` | ||
| fi | ||
| kubectl logs $sql_driver | egrep 'Name: Michael' | ||
| status_3_spark_sql=$(echo $?) | ||
| if [ "`kubectl get pods | grep spark-sql-test | awk '{print $1}'`" != "" ]; then | ||
| kubectl delete pods $sql_driver | ||
| fi | ||
| fi | ||
| if [ $status_4_spark_gbt -ne 0 ]; then | ||
| echo "################## start spark gbt" | ||
| echo "example.4 spark gbt" | ||
| bash run_spark_gbt.sh $IMAGE | ||
| if [ "`kubectl get pods | grep spark-gbt-example-test | awk '{print $1}'`" != "" ]; then | ||
| gbt_example_driver=`kubectl get pods | grep spark-gbt-example-test | awk '{print $1}' | grep driver` | ||
| fi | ||
| kubectl logs $gbt_example_driver | egrep 'Training data results' | ||
| status_4_spark_gbt=$(echo $?) | ||
| if [ "`kubectl get pods | grep spark-gbt-example-test | awk '{print $1}'`" != "" ]; then | ||
| kubectl delete pods $gbt_example_driver | ||
| fi | ||
| fi | ||
| if [ $status_5_spark_gbt_criteo -ne 0 ]; then | ||
| echo "################## start spark gbt criteo" | ||
| echo "example.5 spark gbt criteo" | ||
| bash run_spark_gbt_criteo.sh $IMAGE | ||
| if [ "`kubectl get pods | grep spark-gbt-criteo-test | awk '{print $1}'`" != "" ]; then | ||
| gbt_criteo_driver=`kubectl get pods | grep spark-gbt-criteo-test | grep driver | awk '{print $1}'` | ||
| fi | ||
| kubectl logs $gbt_criteo_driver | egrep 'end time is' | ||
| status_5_spark_gbt_criteo=$(echo $?) | ||
| if [ "`kubectl get pods | grep spark-gbt-criteo-test | awk '{print $1}'`" != "" ]; then | ||
| kubectl delete pods $gbt_criteo_driver | ||
| fi | ||
| fi | ||
| if [ $status_6_spark_tpch -ne 0 ]; then | ||
| echo "################## start spark tpch" | ||
| echo "example.6 spark tpch" | ||
| bash run_spark_tpch.sh $IMAGE | ||
| if [ "`kubectl get pods | grep spark-tpch-test | awk '{print $1}'`" != "" ]; then | ||
| tpch_driver=`kubectl get pods | grep spark-tpch-test | awk '{print $1}' | grep driver` | ||
| fi | ||
| kubectl logs $tpch_driver | egrep '22 finished-' | ||
| status_6_spark_tpch=$(echo $?) | ||
| if [ "`kubectl get pods | grep spark-tpch-test | awk '{print $1}'`" != "" ]; then | ||
| kubectl delete pods $tpch_driver | ||
| fi | ||
| fi | ||
| if [ $status_7_pyspark_sklearn -ne 0 ]; then | ||
| echo "################## start pyspark sklearn" | ||
| echo "example.7 pyspark sklearn" | ||
| bash run_pyspark_sklearn_example.sh $IMAGE | ||
| if [ "`kubectl get pods | grep pyspark-sklearn-test | awk '{print $1}'`" != "" ]; then | ||
| sklearn_driver=`kubectl get pods | grep pyspark-sklearn-test | awk '{print $1}' | grep driver` | ||
| fi | ||
| kubectl logs $sklearn_driver | egrep 'mean_squared_error' | ||
| status_7_pyspark_sklearn=$(echo $?) | ||
| if [ "`kubectl get pods | grep pyspark-sklearn-test | awk '{print $1}'`" != "" ]; then | ||
| kubectl delete pods $sklearn_driver | ||
| fi | ||
| fi | ||
| if [ $status_8_pyspark_sql -ne 0 ]; then | ||
| echo "################## start pyspark SQL example" | ||
| echo "example.8 pyspark sql" | ||
| bash run_pyspark_sql_example.sh $IMAGE | ||
| if [ "`kubectl get pods | grep pyspark-pysql-test | awk '{print $1}'`" != "" ]; then | ||
| sql_driver=`kubectl get pods | grep pyspark-pysql-test | awk '{print $1}' | grep driver` | ||
| fi | ||
| kubectl logs $sql_driver | egrep 'Example API finished' | ||
| status_8_pyspark_sql=$(echo $?) | ||
| if [ "`kubectl get pods | grep pyspark-pysql-test | awk '{print $1}'`" != "" ]; then | ||
| kubectl delete pods $sql_driver | ||
| fi | ||
| fi | ||
| if [ $status_9_pyspark_tpch -ne 0 ]; then | ||
| echo "################## start pyspark tpch" | ||
| echo "example.9 pyspark tpch" | ||
| bash run_pyspark_tpch.sh $IMAGE | ||
| if [ "`kubectl get pods | grep pyspark-tpch-test | awk '{print $1}'`" != "" ]; then | ||
| pytpch_driver=`kubectl get pods | grep pyspark-tpch-test | awk '{print $1}' | grep driver` | ||
| fi | ||
| kubectl logs $pytpch_driver | egrep 'total time is' | ||
| status_9_pyspark_tpch=$(echo $?) | ||
| if [ "`kubectl get pods | grep pyspark-tpch-test | awk '{print $1}'`" != "" ]; then | ||
| kubectl delete pods $pytpch_driver | ||
| fi | ||
| fi | ||
| if [ $status_10_spark_lgbm -ne 0 ]; then | ||
| echo "################## start spark lgbm" | ||
| echo "example.10 spark lgbm" | ||
| bash run_spark_lgbm.sh $IMAGE | ||
| if [ "`kubectl get pods | grep spark-lgbm-test | awk '{print $1}'`" != "" ]; then | ||
| lgbm_driver=`kubectl get pods | grep spark-lgbm-test | awk '{print $1}' | grep driver` | ||
| fi | ||
| kubectl logs $lgbm_driver | egrep 'acc:' | ||
| status_10_spark_lgbm=$(echo $?) | ||
| if [ "`kubectl get pods | grep spark-lgbm-test | awk '{print $1}'`" != "" ]; then | ||
| kubectl delete pods $lgbm_driver | ||
| fi | ||
| fi | ||
| echo "status_1_spark_pi $status_1_spark_pi" | ||
| echo "status_2_spark_lr $status_2_spark_lr" | ||
| echo "status_3_spark_sql $status_3_spark_sql" | ||
| echo "status_4_spark_gbt $status_4_spark_gbt" | ||
| echo "status_5_spark_gbt_criteo $status_5_spark_gbt_criteo" | ||
| echo "status_6_spark_tpch $status_6_spark_tpch" | ||
| echo "status_7_pyspark_sklearn $status_7_pyspark_sklearn" | ||
| echo "status_8_pyspark_sql $status_8_pyspark_sql" | ||
| echo "status_9_pyspark_tpch $status_9_pyspark_tpch" | ||
| echo "status_10_spark_lgbm $status_10_spark_lgbm" | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.