add wedpr-model config generator

wedpr-builder/conf/config-example.toml

@@ -5,6 +5,9 @@ binary_path = "bin/"
 wedpr_site_dist_path = "wedpr-site/dist/"
 # the dist path for wedpr-dir
 wedpr_pir_dist_path = "wedpr-pir/dist/"
+# the wedpr model source path,
+# pls clone from https://github.com/WeBankBlockchain/WeDPR-Component
+wedpr_model_source_path = "WeDPR-Component/python/"
 # use docker mode or not
 docker_mode = true
 deploy_dir = "wedpr-example"
@@ -69,6 +72,7 @@ psi_api_token = "wedpr_psi_api_token_agency0"
         home = "/user/wedpr/agency0"
         name_node = "127.0.0.1"
         name_node_port = 9000
+        webfs_port = 50700
         token = ""
         # enable auth or not, default is false
         enable_krb5_auth = false
@@ -82,6 +86,8 @@ psi_api_token = "wedpr_psi_api_token_agency0"
         krb5_conf_path = "conf/krb5.conf"
         # keytab path
         krb5_keytab_path = ""
+        # auth host name override
+        auth_host_name_override = ""
 
     # the agency mysql configuration
     [agency.mysql]
@@ -103,13 +109,17 @@ psi_api_token = "wedpr_psi_api_token_agency0"
         # the pir deploy ip
         deploy_ip=["127.0.0.1:2"]
         # the server start port
-        server_start_port = "27000"
+        server_start_port = "17000"
+    # configuration for wedpr model
+    [agency.model]
+        deploy_ip = ["127.0.0.1:2"]
+        server_start_port = "18000"
 
     # configuration for wedpr worker
     [agency.jupyter_worker]
         deploy_ip = ["127.0.0.1:2"]
         # the server start port
-        server_start_port = "28000"
+        server_start_port = "19000"
 
     # configuration for the wedpr nodes
     [[agency.node]]
@@ -193,6 +203,7 @@ psi_api_token = "wedpr_psi_api_token_agency1"
         home = "/user/wedpr/agency1"
         name_node = "127.0.0.1"
         name_node_port = 9000
+        webfs_port = 50700
         token = ""
         # enable auth or not, default is false
         enable_krb5_auth = false
@@ -204,6 +215,9 @@ psi_api_token = "wedpr_psi_api_token_agency1"
         ccache_path = "/tmp/krb5cc_ppc_node"
         # the krb5 conf path
         krb5_conf_path = "conf/krb5.conf"
+        # auth host name override
+        auth_host_name_override = ""
+
     # the agency mysql configuration
     [agency.mysql]
         host = "127.0.0.1"
@@ -217,20 +231,25 @@ psi_api_token = "wedpr_psi_api_token_agency1"
         # the site deploy ip
         deploy_ip=["127.0.0.1:2"]
         # the server port
-        server_start_port = "16000"
+        server_start_port = "26000"
 
     # configuration for wedpr pir
     [agency.pir]
         # the pir deploy ip
         deploy_ip=["127.0.0.1:2"]
         # the server start port
-        server_start_port = "27100"
+        server_start_port = "27000"
+
+    # configuration for wedpr model
+    [agency.model]
+        deploy_ip = ["127.0.0.1:2"]
+        server_start_port = "28000"
 
     # configuration for wedpr worker
     [agency.jupyter_worker]
         deploy_ip = ["127.0.0.1:2"]
         # the server start port
-        server_start_port = "28100"
+        server_start_port = "29000"
 
     # configuration for the wedpr nodes
     [[agency.node]]

wedpr-builder/wedpr_builder/common/constant.py

@@ -18,10 +18,11 @@ class ServiceInfo:
     gateway_service_type = "wedpr-gateway"
     wedpr_site_service = "wedpr-site"
     wedpr_pir_service = "wedpr-pir"
+    wedpr_model_service = "wedpr-model"
     wedpr_jupyter_worker_service = "wedpr-jupyter-worker"
     supported_service_type = [node_service_type, gateway_service_type,
                               wedpr_site_service, wedpr_pir_service,
-                              wedpr_jupyter_worker_service]
+                              wedpr_jupyter_worker_service, wedpr_model_service]
 
 
 def get_abs_path(file_path, tpl_abs_path="wedpr_builder/tpl/"):
@@ -54,6 +55,11 @@ class ConfigInfo:
     wedpr_pir_config_path = get_abs_path("pir/conf")
     pir_config_list = ["application-wedpr.properties", "wedpr.properties"]
 
+    wedpr_model_config_path = get_abs_path("model/conf")
+    wedpr_model_config_list = ["application.yml"]
+    wedpr_model_common_dir = "ppc_common"
+    wedpr_model_source_dir = "ppc_model"
+
     wedpr_jupyter_worker_config_path = get_abs_path("worker/conf")
     jupyter_config_list = ["application-wedpr.properties", "wedpr.properties"]
 
@@ -72,6 +78,7 @@ class ConfigProperities:
     USER_JWT_SESSION = "USER_JWT_SESSION_KEY"
     # the mysql configuration
     MYSQL_URL = "MYSQL_URL"
+    SQLALCHEMY_URL = "SQLALCHEMY_URL"
     MYSQL_USER = "MYSQL_USER"
     MYSQL_PASSWORD = "MYSQL_PASSWORD"
     # the blockchain configuration
@@ -92,6 +99,9 @@ class ConfigProperities:
     HDFS_USER = "HDFS_USER"
     HDFS_HOME = "HDFS_HOME"
     HDFS_ENTRYPOINT = "HDFS_ENTRYPOINT"
+    HDFS_WEBFS_ENTRYPOINT = "HDFS_WEBFS_ENTRYPOINT"
     # the hdfs auth configuration
     HDFS_ENABLE_AUTH = "HDFS_ENABLE_AUTH"
     HDFS_AUTH_PRINCIPAL = "HDFS_AUTH_PRINCIPAL"
+    HDFS_AUTH_PASSWORD = "HDFS_AUTH_PASSWORD"
+    HDFS_HOSTNAME_OVERRIDE = "HDFS_HOSTNAME_OVERRIDE"

wedpr-builder/wedpr_builder/common/utilities.py

@@ -155,7 +155,6 @@ def substitute_configurations(config_properities: {}, config_file: str):
         command = "sed -i %s 's/%s/%s/g' %s" % \
                   (option, config_key_var,
                    value, config_file)
-        log_debug(f"* to execute command: {command}")
         (ret, output) = execute_command_and_getoutput(command)
         if ret is False:
             raise Exception(

wedpr-builder/wedpr_builder/config/wedpr_deploy_config.py

@@ -11,12 +11,14 @@ def __init__(self, node_must_exists: bool = False,
                  gateway_must_exists: bool = False,
                  site_must_exists: bool = False,
                  pir_must_exists: bool = False,
-                 jupyter_must_exists: bool = False):
+                 jupyter_must_exists: bool = False,
+                 model_must_exists: bool = False):
         self.node_must_exists = node_must_exists
         self.gateway_must_exists = gateway_must_exists
         self.site_must_exists = site_must_exists
         self.pir_must_exists = pir_must_exists
         self.jupyter_must_exists = jupyter_must_exists
+        self.model_must_exists = model_must_exists
 
 
 class PeerInfo:
@@ -30,11 +32,19 @@ def __repr__(self):
 
 
 class EnvConfig:
-    def __init__(self, config, section_name: str):
+    def __init__(self, config, section_name: str, component_switch: ComponentSwitch):
         self.config = config
         self.section_name = section_name
+        self.component_switch = component_switch
+        # docker_mode
+        self.docker_mode = utilities.get_value(
+            self.config,
+            self.section_name,
+            "docker_mode", True, True)
         self.binary_path = utilities.get_value(
-            self.config, self.section_name, "binary_path", None, True)
+            self.config, self.section_name, "binary_path", None,
+            (not self.docker_mode and
+             (self.component_switch.gateway_must_exists or self.component_switch.node_must_exists)))
         self.deploy_dir = utilities.get_value(
             self.config, self.section_name, "deploy_dir", None, True)
         # zone
@@ -43,23 +53,27 @@ def __init__(self, config, section_name: str):
         # wedpr_site_dist_path
         self.wedpr_site_dist_path = utilities.get_value(
             self.config, self.section_name,
-            "wedpr_site_dist_path", None, True)
+            "wedpr_site_dist_path", None,
+            (not self.docker_mode and self.component_switch.site_must_exists))
         # wedpr_pir_dist_path
         self.wedpr_pir_dist_path = utilities.get_value(
             self.config, self.section_name,
-            "wedpr_pir_dist_path", None, True)
-        # docker_mode
-        self.docker_mode = utilities.get_value(
-            self.config,
-            self.section_name,
-            "docker_mode", True, True)
+            "wedpr_pir_dist_path", None,
+            (not self.docker_mode and self.component_switch.pir_must_exists))
+        # model path
+        self.wedpr_model_source_path = utilities.get_value(
+            self.config, self.section_name, "wedpr_model_source_path", None,
+            (not self.docker_mode and self.component_switch.model_must_exists))
 
     # Note: jupyter only use docker mode
+
     def get_dist_path_by_service_type(self, service_type: str) -> str:
         if service_type == constant.ServiceInfo.wedpr_site_service:
             return self.wedpr_site_dist_path
         if service_type == constant.ServiceInfo.wedpr_pir_service:
             return self.wedpr_pir_dist_path
+        if service_type == constant.ServiceInfo.wedpr_model_service:
+            return self.wedpr_model_source_path
         return None
 
     def __repr__(self):
@@ -226,6 +240,10 @@ def to_properties(self) -> {}:
         properties = {}
         mysql_url = f"jdbc:mysql://{self.host}:{self.port}/{self.database}?characterEncoding=UTF-8&allowMultiQueries=true"
         properties.update({constant.ConfigProperities.MYSQL_URL: mysql_url})
+        # the sql-alchemy url
+        sql_alchemy_url = f"mysql://{self.user}:{self.password}@{self.host}:{self.port}/{self.database}?autocommit=true&charset=utf8mb4"
+        properties.update(
+            {constant.ConfigProperities.SQLALCHEMY_URL: sql_alchemy_url})
         properties.update({constant.ConfigProperities.MYSQL_USER: self.user})
         properties.update(
             {constant.ConfigProperities.MYSQL_PASSWORD: self.password})
@@ -302,6 +320,8 @@ def __init__(self, config, config_section: str, must_exist: bool):
             self.config, "name_node", None, must_exist, config_section)
         self.name_node_port = utilities.get_item_value(
             self.config, "name_node_port", None, must_exist, config_section)
+        self.webfs_port = utilities.get_item_value(
+            self.config, "webfs_port", None, must_exist, config_section)
         self.token = utilities.get_item_value(
             self.config, "token", "", False, config_section)
         # enable auth or not
@@ -328,6 +348,8 @@ def __init__(self, config, config_section: str, must_exist: bool):
             "conf/krb5.conf", enable_krb5_auth, config_section)
         self.krb5_keytab_path = utilities.get_item_value(
             self.config, "krb5_keytab_path", None, enable_krb5_auth, config_section)
+        self.auth_host_name_override = utilities.get_item_value(
+            self.config, "auth_host_name_override", None, enable_krb5_auth, config_section)
 
     def __repr__(self):
         return f"**HDFSStorageConfig: user: {self.user}, " \
@@ -342,12 +364,25 @@ def to_properties(self) -> {}:
         props = {}
         props.update({constant.ConfigProperities.HDFS_USER: self.user})
         props.update({constant.ConfigProperities.HDFS_HOME: self.home})
+        # rpc entrypoint
         hdfs_url = f"hdfs://{self.name_node}:{self.name_node_port}"
         props.update({constant.ConfigProperities.HDFS_ENTRYPOINT: hdfs_url})
+        # webfs entrypoint
+        webfs_url = f"http://{self.name_node}:{self.webfs_port}"
+        props.update(
+            {constant.ConfigProperities.HDFS_WEBFS_ENTRYPOINT: webfs_url})
+        # enable auth or not
         props.update(
             {constant.ConfigProperities.HDFS_ENABLE_AUTH: self.enable_krb5_auth_str})
+        # auth principal
         props.update(
             {constant.ConfigProperities.HDFS_AUTH_PRINCIPAL: self.auth_principal})
+        # password
+        props.update(
+            {constant.ConfigProperities.HDFS_AUTH_PASSWORD: self.auth_password})
+        # auth_hostname_override
+        props.update(
+            {constant.ConfigProperities.HDFS_HOSTNAME_OVERRIDE: self.auth_host_name_override})
         return props
 
 
@@ -535,6 +570,15 @@ def __init__(self, config, env_config: EnvConfig,
             constant.ConfigInfo.wedpr_pir_config_path,
             constant.ConfigInfo.pir_config_list)
 
+        # load the model service config
+        self.model_service_config = self.__load_service_config__(
+            "[agency.model]", "model",
+            self.component_switch.model_must_exists,
+            constant.ServiceInfo.wedpr_model_service,
+            constant.ConfigInfo.wedpr_model_config_path,
+            constant.ConfigInfo.wedpr_model_config_list
+        )
+
         # load the jupyter_worker config
         self.jupyter_worker_config = self.__load_service_config__(
             "[agency.jupyter_worker]", "jupyter_worker",
@@ -616,6 +660,19 @@ def to_properties(self) -> {}:
             {constant.ConfigProperities.PSI_API_TOKEN: self.psi_api_token})
         return props
 
+    def get_wedpr_model_properties(self, deploy_ip: str, node_index: int) -> {}:
+        props = self.to_properties()
+        # the zone config
+        props.update(self.env_config.to_properties())
+        # the sql config
+        props.update(self.sql_storage_config.to_properties())
+        # the hdfs config
+        props.update(self.hdfs_storage_config.to_properties())
+        # the service config
+        props.update(self.model_service_config.to_properties(
+            deploy_ip, node_index))
+        return props
+
     def get_wedpr_site_properties(self, deploy_ip: str, node_index: int) -> {}:
         """
         get the site config properties
@@ -697,7 +754,7 @@ def __init__(self, config, component_switch: ComponentSwitch):
         self.sm_crypto = utilities.get_value(
             self.config, crypto_section, "sm_crypto", False, False)
         utilities.log_debug("load the crypto config success")
-        self.env_config = EnvConfig(self.config, "env")
+        self.env_config = EnvConfig(self.config, "env", self.component_switch)
         # the blockchain config
         self.blockchain_config = BlockchainConfig(self.config, "blockchain")
         # load the agency config

wedpr-builder/wedpr_builder/controller/commandline_helper.py

@@ -9,6 +9,7 @@
 from wedpr_builder.generator.wedpr_service_generator import WedprSiteServiceGenerator
 from wedpr_builder.generator.wedpr_service_generator import WedprPirServiceGenerator
 from wedpr_builder.generator.wedpr_service_generator import WedprJupyterWorkerServiceGenerator
+from wedpr_builder.generator.wedpr_service_generator import WedprModelServiceGenerator
 from argparse import RawTextHelpFormatter
 import sys
 import toml
@@ -21,7 +22,8 @@ def parse_command():
                 "python3 build_wedpr.py -t wedpr-node\n " \
                 "* generate gateway config:\t python3 build_wedpr.py -t wedpr-gateway\n " \
                 "* generate wedpr-site config:\t python3 build_wedpr.py -t wedpr-site\n " \
-                "* generate wedpr-pir config:\t python3 build_wedpr.py -t wedpr-pir\n " \
+                "* generate wedpr-pir config:\t python3 build_wedpr.py -t wedpr-pir\n" \
+                "* generate wedpr-model service config:\t python3 build_wedpr.py -t wedpr-model\n " \
                 "* generate wedpr-jupyter-worker config:\t python3 build_wedpr.py -t wedpr-jupyter-worker\n " \
                 "* generate gateway config:\t python3 build_wedpr.py -o genconfig -c config.toml -t wedpr-gateway -d wedpr-generated\n " \
                 "* generate node config:\t python3 build_wedpr.py -o genconfig -c config.toml -t wedpr-node -d wedpr-generated"
@@ -93,6 +95,13 @@ def generate_node_config(args, toml_config):
         config = WeDPRDeployConfig(toml_config, component_switch)
         pir_generator = WedprPirServiceGenerator(config, args.output)
         pir_generator.generate_config()
+    # the model service generator
+    if service_type == constant.ServiceInfo.wedpr_model_service:
+        component_switch = ComponentSwitch(model_must_exists=True)
+        config = WeDPRDeployConfig(toml_config, component_switch)
+        model_service_generator = WedprModelServiceGenerator(
+            config, args.output)
+        model_service_generator.generate_config()
     # the jupyter worker config generator
     if service_type == constant.ServiceInfo.wedpr_jupyter_worker_service:
         component_switch = ComponentSwitch(jupyter_must_exists=True)