add wedpr-model config generator

docker-files/base/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM ubuntu:18.04 as wedpr_image
 
 # install python and java
 RUN apt-get update && apt-get install -y tzdata language-pack-zh* \

docker-files/jupyter/Dockerfile

@@ -1,4 +1,4 @@
-FROM wedpr-image:latest
+FROM wedpr-image:latest as  wedpr-jupyter-image
 
 ENV LANG zh_CN.UTF-8
 ENV LANGUAGE zh_CN.UTF-8

docker-files/site/Dockerfile

@@ -16,23 +16,23 @@ RUN git clone  https://github.com/WeBankBlockchain/WeDPR.git -b ${SOURCE} \
 FROM wedpr-jupyter-image:latest as wedpr-jupyter-worker
 LABEL maintainer [email protected]
 
-RUN mkdir -p /data/app/
-COPY --from=builder /WeDPR/wedpr-worker/dist/ /data/app/wedpr-worker
-ENTRYPOINT ["/bin/bash"]
+RUN mkdir -p /data/home/wedpr
+COPY --from=builder /WeDPR/wedpr-worker/dist/ /data/home/wedpr/wedpr-worker
+ENTRYPOINT ["/bin/bash", "/data/home/wedpr/wedpr-worker/start.sh", "true"]
 
 # the wedpr-pir
 FROM wedpr-image:latest as wedpr-pir
 LABEL maintainer [email protected]
 
-RUN mkdir -p /data/app/
+RUN mkdir -p /data/home/wedpr
 COPY --from=builder /WeDPR/wedpr-pir/dist/ /data/app/wedpr-pir
-ENTRYPOINT ["/bin/bash"]
+ENTRYPOINT ["/bin/bash", "/data/home/wedpr/wedpr-pir/start.sh", "true"]
 
 # the wedpr-site
 FROM wedpr-image:latest as wedpr-site
 LABEL maintainer [email protected]
 
-RUN mkdir -p /data/app/
-COPY --from=builder /WeDPR/wedpr-site/dist/ /data/app/wedpr-site
+RUN mkdir -p /data/home/wedpr
+COPY --from=builder /WeDPR/wedpr-site/dist/ /data/home/wedpr/wedpr-site
 # TODO: fetch the web package from github tag
-ENTRYPOINT ["/bin/bash"]
+ENTRYPOINT ["/bin/bash", "/data/home/wedpr/wedpr-site/start.sh", "true"]

wedpr-builder/conf/config-example.toml

@@ -1,15 +1,31 @@
 [env]
+#### define the binary path ###
 # the binary path for ppc-pro-node and ppc-gateway-service
-binary_path = "bin/"
+binary_path = "/data/home/wedpr/WeDPR-Component/cpp/build/bin"
 # the dist path for wedpr-site
-wedpr_site_dist_path = "wedpr-site/dist/"
+wedpr_site_dist_path = "/data/home/wedpr/WeDPR/wedpr-site/dist/"
 # the dist path for wedpr-dir
-wedpr_pir_dist_path = "wedpr-pir/dist/"
+wedpr_pir_dist_path = "/data/home/wedpr/WeDPR/wedpr-pir/dist/"
+# the wedpr model source path,
+# pls clone from https://github.com/WeBankBlockchain/WeDPR-Component
+wedpr_model_source_path = "/data/home/wedpr/WeDPR-Component/python/"
+
 # use docker mode or not
 docker_mode = true
+
+#### define the docker images desc ###
+wedpr_gateway_service_image_desc = "wedpr-gateway-service:latest"
+wedpr_node_service_image_desc = "wedpr-pro-node-service:latest"
+wedpr_mpc_service_image_desc = "wedpr-mpc-service:latest"
+wedpr_jupyter_worker_image_desc = "wedpr-jupyter-worker:latest"
+wedpr_model_image_desc = "wedpr-model-service:latest"
+wedpr_site_image_desc = "wedpr-site:latest"
+wedpr_pir_image_desc = "wedpr-pir:latest"
+
+
 deploy_dir = "wedpr-example"
 # the wedpr zone(used to distinguish different privacy computing environments)
-zone = "wedpr.zone.example"
+zone = "wedpr.zone.default"
 
 # the blockchain config
 [blockchain]
@@ -69,6 +85,7 @@ psi_api_token = "wedpr_psi_api_token_agency0"
         home = "/user/wedpr/agency0"
         name_node = "127.0.0.1"
         name_node_port = 9000
+        webfs_port = 50700
         token = ""
         # enable auth or not, default is false
         enable_krb5_auth = false
@@ -82,6 +99,8 @@ psi_api_token = "wedpr_psi_api_token_agency0"
         krb5_conf_path = "conf/krb5.conf"
         # keytab path
         krb5_keytab_path = ""
+        # auth host name override
+        auth_host_name_override = ""
 
     # the agency mysql configuration
     [agency.mysql]
@@ -103,13 +122,17 @@ psi_api_token = "wedpr_psi_api_token_agency0"
         # the pir deploy ip
         deploy_ip=["127.0.0.1:2"]
         # the server start port
-        server_start_port = "27000"
+        server_start_port = "17000"
+    # configuration for wedpr model
+    [agency.model]
+        deploy_ip = ["127.0.0.1:2"]
+        server_start_port = "18000"
 
     # configuration for wedpr worker
     [agency.jupyter_worker]
-        deploy_ip = ["127.0.0.1:2"]
+        deploy_ip = ["127.0.0.1:1"]
         # the server start port
-        server_start_port = "28000"
+        server_start_port = "19000"
 
     # configuration for the wedpr nodes
     [[agency.node]]
@@ -193,6 +216,7 @@ psi_api_token = "wedpr_psi_api_token_agency1"
         home = "/user/wedpr/agency1"
         name_node = "127.0.0.1"
         name_node_port = 9000
+        webfs_port = 50700
         token = ""
         # enable auth or not, default is false
         enable_krb5_auth = false
@@ -204,6 +228,9 @@ psi_api_token = "wedpr_psi_api_token_agency1"
         ccache_path = "/tmp/krb5cc_ppc_node"
         # the krb5 conf path
         krb5_conf_path = "conf/krb5.conf"
+        # auth host name override
+        auth_host_name_override = ""
+
     # the agency mysql configuration
     [agency.mysql]
         host = "127.0.0.1"
@@ -217,20 +244,25 @@ psi_api_token = "wedpr_psi_api_token_agency1"
         # the site deploy ip
         deploy_ip=["127.0.0.1:2"]
         # the server port
-        server_start_port = "16000"
+        server_start_port = "26000"
 
     # configuration for wedpr pir
     [agency.pir]
         # the pir deploy ip
         deploy_ip=["127.0.0.1:2"]
         # the server start port
-        server_start_port = "27100"
+        server_start_port = "27000"
+
+    # configuration for wedpr model
+    [agency.model]
+        deploy_ip = ["127.0.0.1:2"]
+        server_start_port = "28000"
 
     # configuration for wedpr worker
     [agency.jupyter_worker]
-        deploy_ip = ["127.0.0.1:2"]
+        deploy_ip = ["127.0.0.1:1"]
         # the server start port
-        server_start_port = "28100"
+        server_start_port = "29000"
 
     # configuration for the wedpr nodes
     [[agency.node]]

wedpr-builder/wedpr_builder/common/constant.py

@@ -18,10 +18,14 @@ class ServiceInfo:
     gateway_service_type = "wedpr-gateway"
     wedpr_site_service = "wedpr-site"
     wedpr_pir_service = "wedpr-pir"
+    wedpr_mpc_service = "wedpr-mpc-service"
+    wedpr_model_service = "wedpr-model"
     wedpr_jupyter_worker_service = "wedpr-jupyter-worker"
     supported_service_type = [node_service_type, gateway_service_type,
                               wedpr_site_service, wedpr_pir_service,
-                              wedpr_jupyter_worker_service]
+                              wedpr_jupyter_worker_service,
+                              wedpr_model_service,
+                              wedpr_mpc_service]
 
 
 def get_abs_path(file_path, tpl_abs_path="wedpr_builder/tpl/"):
@@ -54,8 +58,29 @@ class ConfigInfo:
     wedpr_pir_config_path = get_abs_path("pir/conf")
     pir_config_list = ["application-wedpr.properties", "wedpr.properties"]
 
+    wedpr_model_config_path = get_abs_path("model/conf")
+    wedpr_model_config_list = ["application.yml"]
+    wedpr_model_common_dir = "ppc_common"
+    wedpr_model_source_dir = "ppc_model"
+
     wedpr_jupyter_worker_config_path = get_abs_path("worker/conf")
     jupyter_config_list = ["application-wedpr.properties", "wedpr.properties"]
+    # the docker config
+    docker_tpl_path = get_abs_path("docker/")
+    default_docker_work_dir = "/data/home/wedpr/"
+    wedpr_worker_docker_dir = "wedpr-worker"
+    wedpr_pir_docker_dir = "wedpr-pir"
+    wedpr_site_docker_dir = "wedpr-site"
+
+    wedpr_gateway_service_dir = "wedpr-gateway-service"
+    wedpr_node_service_dir = "wedpr-pro-node-service"
+    wedpr_mpc_service_dir = "wedpr-mpc-service"
+    docker_file_list = ["create_docker.sh",
+                        "start_docker.sh", "stop_docker.sh"]
+
+    @staticmethod
+    def get_docker_path(file_path: str):
+        return os.path.join(ConfigInfo.default_docker_work_dir, file_path)
 
 
 class CommandInfo:
@@ -72,6 +97,7 @@ class ConfigProperities:
     USER_JWT_SESSION = "USER_JWT_SESSION_KEY"
     # the mysql configuration
     MYSQL_URL = "MYSQL_URL"
+    SQLALCHEMY_URL = "SQLALCHEMY_URL"
     MYSQL_USER = "MYSQL_USER"
     MYSQL_PASSWORD = "MYSQL_PASSWORD"
     # the blockchain configuration
@@ -92,6 +118,25 @@ class ConfigProperities:
     HDFS_USER = "HDFS_USER"
     HDFS_HOME = "HDFS_HOME"
     HDFS_ENTRYPOINT = "HDFS_ENTRYPOINT"
+    HDFS_WEBFS_ENTRYPOINT = "HDFS_WEBFS_ENTRYPOINT"
     # the hdfs auth configuration
     HDFS_ENABLE_AUTH = "HDFS_ENABLE_AUTH"
     HDFS_AUTH_PRINCIPAL = "HDFS_AUTH_PRINCIPAL"
+    HDFS_AUTH_PASSWORD = "HDFS_AUTH_PASSWORD"
+    HDFS_HOSTNAME_OVERRIDE = "HDFS_HOSTNAME_OVERRIDE"
+    #### the docker related vars ###
+    # specify the docker image desc, include: org/docker_name:version
+    WEDPR_IMAGE_DESC = "WEDPR_IMAGE_DESC"
+    # specify the conf path to mount
+    WEDPR_CONFIG_DIR = "WEDPR_CONFIG_DIR"
+    # specify the mounted docker conf path
+    DOCKER_CONF_PATH = "DOCKER_CONF_PATH"
+    # specify the log path to mount
+    WEDPR_LOG_DIR = "WEDPR_LOG_DIR"
+    # specify the mounted docker log path
+    DOCKER_LOG_PATH = "DOCKER_LOG_PATH"
+    # specify the exposed docker port list
+    WEDPR_DOCKER_EXPORSE_PORT_LIST = "WEDPR_DOCKER_EXPORSE_PORT_LIST"
+    # the created docker name
+    WEDPR_DOCKER_NAME = "WEDPR_DOCKER_NAME"
+    EXTENDED_MOUNT_CONF = "EXTENDED_MOUNT_CONF"

wedpr-builder/wedpr_builder/common/utilities.py

@@ -152,10 +152,10 @@ def substitute_configurations(config_properities: {}, config_file: str):
             value = config_value.replace("/", "\/")
             value = value.replace("&", '\&')
         config_key_var = '${%s}' % config_key
-        command = "sed -i %s 's/%s/%s/g' %s" % \
+        command = "sed -i %s 's/%s/%s/g' %s && rm -rf %s.bkp" % \
                   (option, config_key_var,
-                   value, config_file)
-        log_debug(f"* to execute command: {command}")
+                   value, config_file, config_file)
+
         (ret, output) = execute_command_and_getoutput(command)
         if ret is False:
             raise Exception(