add mpc config generator (#194)

* add upload wedpr-sol

* remove rpc disable ssl

* add mpc config generator

.github/workflows/release.yml

@@ -31,6 +31,7 @@ jobs:
       - name: prepare wedpr-builder tgz
         run: |
           tar -cvzf wedpr-builder.tar.gz wedpr-builder
+          tar -cvzf wedpr-sol.tar.gz wedpr-sol
       - name: Upload wedpr-builder
         uses: svenstaro/upload-release-action@v1-release
         with:
@@ -39,6 +40,14 @@ jobs:
           asset_name: wedpr-builder.tar.gz
           tag: ${{ github.ref }}
           overwrite: true
+      - name: Upload wedpr-sol
+        uses: svenstaro/upload-release-action@v1-release
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: wedpr-sol.tar.gz
+          asset_name: wedpr-sol.tar.gz
+          tag: ${{ github.ref }}
+          overwrite: true
 
   upload-web:
     name: upload-web

python/jupyter/setup.py

@@ -27,7 +27,7 @@ def run(self):
 setup_args = dict(
     name='wedpr-authenticator',
     packages=find_packages(),
-    version="1.0.0.dev-20241124",
+    version="3.0.0-20241124",
     description="WeDPR Authenticator: Authenticate JupyterHub users with wedpr providers",
     long_description_content_type="text/markdown",
     author="WeDPR Team",

python/mpc_generator/setup.py

@@ -20,7 +20,7 @@ def run(self):
 setup_args = dict(
     name='wedpr_mpc_generator',
     packages=find_packages(),
-    version="1.0.0.dev-20241206",
+    version="3.0.0-20241206",
     description="wedpr-mpc-generator: The mpc generator for WeDPR",
     long_description_content_type="text/markdown",
     author="WeDPR Development Team",

wedpr-builder/conf/config-example.toml

@@ -2,6 +2,8 @@
 #### define the binary path ###
 # the binary path for ppc-pro-node and ppc-gateway-service
 binary_path = "/data/home/wedpr/WeDPR-Component/cpp/build/bin"
+# the spdz path
+spdz_home = ""
 # the dist path for wedpr-site
 wedpr_site_dist_path = "/data/home/wedpr/WeDPR/wedpr-site/dist/"
 # the dist path for wedpr-dir
@@ -47,16 +49,14 @@ gateway_disable_ssl = false
 gateway_sm_ssl = false
 # the rpc use sm-ssl or not
 rpc_sm_ssl = false
-# the rpc disable ssl or not
-rpc_disable_ssl = true
 # use sm-crypto or not
 sm_crypto = false
 
 [[agency]]
 name = "agency0"
 #the time of the gateway holding and waiting to dispatcher the unsynced task, in minutes
 holding_msg_minutes = 30
-psi_api_token = "wedpr_psi_api_token_agency0"
+wedpr_api_token = ""
 
     [agency.gateway]
     deploy_ip=["127.0.0.1:2"]
@@ -128,12 +128,18 @@ psi_api_token = "wedpr_psi_api_token_agency0"
         deploy_ip = ["127.0.0.1:2"]
         server_start_port = "18000"
 
-    # configuration for wedpr worker
+    # configuration for wedpr jupyter worker
     [agency.jupyter_worker]
         deploy_ip = ["127.0.0.1:1"]
         # the server start port
         server_start_port = "19000"
 
+    # configuration for mpc
+    [agency.mpc]
+        deploy_ip = ["127.0.0.1:1"]
+        # the server start port
+        server_start_port = "20000"
+
     # configuration for the wedpr nodes
     [[agency.node]]
         # disable the ra2018 psi or not, default enable ra2018
@@ -187,7 +193,7 @@ psi_api_token = "wedpr_psi_api_token_agency0"
 name = "agency1"
 #the time of the gateway holding and waiting to dispatcher the unsynced task, in minutes
 holding_msg_minutes = 30
-psi_api_token = "wedpr_psi_api_token_agency1"
+wedpr_api_token = ""
 
     [agency.gateway]
     deploy_ip=["127.0.0.1:2"]
@@ -258,12 +264,18 @@ psi_api_token = "wedpr_psi_api_token_agency1"
         deploy_ip = ["127.0.0.1:2"]
         server_start_port = "28000"
 
-    # configuration for wedpr worker
+    # configuration for wedpr jupyter worker
     [agency.jupyter_worker]
         deploy_ip = ["127.0.0.1:1"]
         # the server start port
         server_start_port = "29000"
 
+    # configuration for mpc
+    [agency.mpc]
+        deploy_ip = ["127.0.0.1:1"]
+        # the server start port
+        server_start_port = "30000"
+
     # configuration for the wedpr nodes
     [[agency.node]]
         # disable the ra2018 psi or not, default enable ra2018
@@ -311,21 +323,4 @@ psi_api_token = "wedpr_psi_api_token_agency1"
             port = 33060
             user = "wedpr"
             password = ""
-            database = ""
-
-        # the hdfs storage config
-        [agency.node.hdfs_storage]
-            user = "root"
-            name_node = "127.0.0.1"
-            name_node_port = 9000
-            token = ""
-            # enable auth or not, default is false
-            enable_krb5_auth = false
-            # the hdfs kerberos auth principal, used when enable_krb5_auth
-            auth_principal = "[email protected]"
-            # the hdfs kerberos auth password, used when enable_krb5_auth
-            auth_password = ""
-            # the ccache path, used when enable_krb5_auth
-            ccache_path = "/tmp/krb5cc_ppc_node"
-            # the krb5 conf path
-            krb5_conf_path = "conf/krb5.conf"
+            database = ""

wedpr-builder/wedpr_builder/common/constant.py

@@ -18,7 +18,7 @@ class ServiceInfo:
     gateway_service_type = "wedpr-gateway"
     wedpr_site_service = "wedpr-site"
     wedpr_pir_service = "wedpr-pir"
-    wedpr_mpc_service = "wedpr-mpc-service"
+    wedpr_mpc_service = "wedpr-mpc"
     wedpr_model_service = "wedpr-model"
     wedpr_jupyter_worker_service = "wedpr-jupyter-worker"
     supported_service_type = [node_service_type, gateway_service_type,
@@ -38,6 +38,7 @@ class ConfigInfo:
     tpl_abs_path = "wedpr_builder/tpl/"
     ppc_gateway_binary_name = "ppc-gateway-service"
     ppc_node_binary_name = "ppc-pro-node"
+    mpc_binary_name = "wedpr-mpc"
     hdfs_key_tab_file_name = "hdfs-wedpr.keytab"
 
     node_config_tpl_path = get_abs_path("config.ini.node")
@@ -81,6 +82,7 @@ class ConfigInfo:
     wedpr_worker_docker_dir = "wedpr-worker"
     wedpr_pir_docker_dir = "wedpr-pir"
     wedpr_site_docker_dir = "wedpr-site"
+    wedpr_mpc_docker_dir = "wedpr-mpc-service"
 
     wedpr_gateway_service_dir = "wedpr-gateway-service"
     wedpr_node_service_dir = "wedpr-pro-node-service"
@@ -100,6 +102,14 @@ class ConfigInfo:
     # the site init conf
     init_tpl_path = get_abs_path("init/")
     init_file_path_list = ["site_init.sh"]
+    # the mpc config
+    mpc_tpl_path = get_abs_path("mpc/")
+    mpc_config_file_list = ["config.ini"]
+    # the scripts
+    scripts_tpl_path = get_abs_path("scripts/")
+    scripts_file_list = ["start.sh", "stop.sh"]
+    # the default path for docker
+    default_spdz_home = "/data/app/wedpr/scripts/wedpr-mpc-no-gateway"
 
     @staticmethod
     def get_docker_path(file_path: str):
@@ -133,19 +143,23 @@ class ConfigProperities:
     WEDPR_RECORDER_CONTRACT_ADDRESS = "WEDPR_RECORDER_CONTRACT_ADDRESS"
     WEDPR_SEQUENCER_CONTRACT_ADDRESS = "WEDPR_SEQUENCER_CONTRACT_ADDRESS"
     # the psi configuration
-    PSI_API_TOKEN = "PSI_API_TOKEN"
+    WEDPR_API_TOKEN = "WEDPR_API_TOKEN"
     # the transport configuration
     WEDPR_NODE_ID = "WEDPR_NODE_ID"
     GATEWAY_TARGET = "GATEWAY_TARGET"
     WEDPR_TRANSPORT_HOST_IP = "WEDPR_TRANSPORT_HOST_IP"
     WEDPR_TRANSPORT_LISTEN_PORT = "WEDPR_TRANSPORT_LISTEN_PORT"
     # the service configuration
     WEDPR_SERVER_LISTEN_PORT = "WEDPR_SERVER_LISTEN_PORT"
+    # the mpc spdz listen port
+    WEDPR_MPC_SPDZ_LISTEN_PORT = "WEDPR_MPC_SPDZ_LISTEN_PORT"
     # the hdfs configuration
     HDFS_USER = "HDFS_USER"
     HDFS_HOME = "HDFS_HOME"
     HDFS_ENTRYPOINT = "HDFS_ENTRYPOINT"
     HDFS_WEBFS_ENTRYPOINT = "HDFS_WEBFS_ENTRYPOINT"
+    HDFS_NAMENODE_HOST = "HDFS_NAMENODE_HOST"
+    HDFS_NAMENODE_PORT = "HDFS_NAMENODE_PORT"
     # the hdfs auth configuration
     HDFS_ENABLE_AUTH = "HDFS_ENABLE_AUTH"
     HDFS_AUTH_PRINCIPAL = "HDFS_AUTH_PRINCIPAL"
@@ -171,3 +185,6 @@ class ConfigProperities:
     # the nginx configuration
     NGINX_BACKEND_SERVER_LIST = "NGINX_BACKEND_SERVER_LIST"
     NGINX_PORT = "NGINX_PORT"
+    # the binary_name
+    BINARY_NAME = "BINARY_NAME"
+    MPC_SPDZ_HOME = "MPC_SPDZ_HOME"

wedpr-builder/wedpr_builder/common/utilities.py

@@ -30,7 +30,7 @@ def log_debug(error_msg):
 
 
 def get_item_value(config, key, default_value, must_exist, desc):
-    if key in config:
+    if config is not None and key in config:
         return config[key]
     if must_exist:
         raise Exception("the value for %s.%s must be set" % (desc, key))

wedpr-builder/wedpr_builder/config/wedpr_deploy_config.py

@@ -98,6 +98,12 @@ def __init__(self, config, section_name: str, component_switch: ComponentSwitch)
         self.wedpr_site_image_desc = utilities.get_value(
             self.config, self.section_name, "wedpr_site_image_desc", None,
             self.component_switch.site_must_exists)
+        # the spdz home
+        self.spdz_home = utilities.get_value(
+            self.config, self.section_name, "spdz_home",
+            constant.ConfigInfo.default_spdz_home, False)
+        if len(self.spdz_home) == 0:
+            self.spdz_home = constant.ConfigInfo.default_spdz_home
 
     # Note: jupyter only use docker mode
     def get_dist_path_by_service_type(self, service_type: str) -> str:
@@ -107,6 +113,8 @@ def get_dist_path_by_service_type(self, service_type: str) -> str:
             return self.wedpr_pir_dist_path
         if service_type == constant.ServiceInfo.wedpr_model_service:
             return self.wedpr_model_source_path
+        if service_type == constant.ServiceInfo.wedpr_mpc_service:
+            return os.path.join(self.binary_path, constant.ConfigInfo.mpc_binary_name)
         return None
 
     def get_image_desc_by_service_name(self, service_type: str) -> str:
@@ -133,6 +141,8 @@ def __repr__(self):
     def to_properties(self) -> {}:
         props = {}
         props.update({constant.ConfigProperities.WEDPR_ZONE: self.zone})
+        props.update(
+            {constant.ConfigProperities.MPC_SPDZ_HOME: self.spdz_home})
         return props
 
 
@@ -409,7 +419,7 @@ def to_properties(self, deploy_ip, node_index: int) -> {}:
         self.server_backend_list.append(
             f"server {deploy_ip}:{server_start_port}")
         # nodeid
-        node_id = f"{self.service_type}-{self.env_config.zone}-node{node_index}"
+        node_id = f"{self.agency}-{self.service_type}-{self.env_config.zone}-{deploy_ip}-node{node_index}"
         props.update({constant.ConfigProperities.WEDPR_NODE_ID: node_id})
         # gateway target
         props.update(
@@ -452,9 +462,15 @@ def to_properties(self, deploy_ip, node_index: int) -> {}:
             # add the SingleJupyterInfo
             self.jupyter_infos.jupyters.append(SingleJupyterInfo(
                 entry_point=entry_point,
-                start_port=begin_port,))
+                start_port=begin_port))
+        if self.service_type == constant.ServiceInfo.wedpr_mpc_service:
+            spdz_listen_port = server_start_port + 2
+            props.update(
+                {constant.ConfigProperities.WEDPR_MPC_SPDZ_LISTEN_PORT: spdz_listen_port})
+            exposed_port_list = f"{exposed_port_list} -p {spdz_listen_port}:{spdz_listen_port}"
         props.update(
             {constant.ConfigProperities.WEDPR_DOCKER_EXPORSE_PORT_LIST: exposed_port_list})
+
         # set the docker name
         docker_name = f"{self.agency}-{self.service_type}-{self.env_config.zone}-node{node_index}"
         props.update(
@@ -526,6 +542,12 @@ def to_properties(self) -> {}:
         # rpc entrypoint
         hdfs_url = f"hdfs://{self.name_node}:{self.name_node_port}"
         props.update({constant.ConfigProperities.HDFS_ENTRYPOINT: hdfs_url})
+        # hdfs_namenode_host
+        props.update(
+            {constant.ConfigProperities.HDFS_NAMENODE_HOST: self.name_node})
+        # hdfs_namenode_port
+        props.update(
+            {constant.ConfigProperities.HDFS_NAMENODE_PORT: self.name_node_port})
         # webfs entrypoint
         webfs_url = f"http://{self.name_node}:{self.webfs_port}"
         props.update(
@@ -697,8 +719,10 @@ def __init__(self, config, env_config: EnvConfig,
         self.holding_msg_minutes = utilities.get_item_value(
             self.config, "holding_msg_minutes", 30, False, self.section_name)
         # the psi api token
-        self.psi_api_token = utilities.get_item_value(
-            self.config, "psi_api_token", "wedpr_psi_api_token", False, self.section_name)
+        self.wedpr_api_token = utilities.get_item_value(
+            self.config, "wedpr_api_token", "", False, self.section_name)
+        if len(self.wedpr_api_token) == 0:
+            self.wedpr_api_token = f"wedpr_api_token_{self.agency_name}"
         # parse the gateway config
         utilities.log_debug("load the gateway config")
         gateway_config_section_name = "[agency.gateway]"
@@ -750,6 +774,13 @@ def __init__(self, config, env_config: EnvConfig,
             constant.ServiceInfo.wedpr_jupyter_worker_service,
             constant.ConfigInfo.wedpr_jupyter_worker_config_path,
             constant.ConfigInfo.jupyter_config_list)
+        # load the mpc config
+        self.mpc_config = self.__load_service_config__(
+            "[agency.mpc]", "mpc",
+            self.component_switch.mpc_service_must_exists,
+            constant.ServiceInfo.wedpr_mpc_service,
+            constant.ConfigInfo.mpc_tpl_path,
+            constant.ConfigInfo.mpc_config_file_list)
 
         # load the jwt user config
         self.user_jwt_config = UserJWTConfig()
@@ -822,7 +853,7 @@ def to_properties(self) -> {}:
         props.update(
             {constant.ConfigProperities.WEDPR_AGENCY: self.agency_name})
         props.update(
-            {constant.ConfigProperities.PSI_API_TOKEN: self.psi_api_token})
+            {constant.ConfigProperities.WEDPR_API_TOKEN: self.wedpr_api_token})
         # EXTENDED_MOUNT_CONF default is empty string
         props.update({constant.ConfigProperities.EXTENDED_MOUNT_CONF: ""})
         return props
@@ -885,10 +916,10 @@ def generate_cpp_component_docker_properties(
             extended_mount_conf = f"{extended_mount_conf} -v {local_mount_dir}:{remote_mount_dir} "
         props.update(
             {constant.ConfigProperities.EXTENDED_MOUNT_CONF: extended_mount_conf})
-        # specify the conf path to mount
-        props.update({constant.ConfigProperities.WEDPR_LOG_DIR: "logs"})
+        # specify the log path to mount
+        props.update({constant.ConfigProperities.WEDPR_LOG_DIR: "log"})
         props.update({constant.ConfigProperities.DOCKER_LOG_PATH:
-                      constant.ConfigInfo.get_docker_path(f"{prefix_path}/logs")})
+                      constant.ConfigInfo.get_docker_path(f"{prefix_path}/log")})
         # set the image desc for docker mode
         image_desc = env_config.get_image_desc_by_service_name(service_type)
         if image_desc is not None:
@@ -980,6 +1011,28 @@ def get_pir_properties(self, deploy_ip: str, node_index: int):
             constant.ConfigInfo.wedpr_pir_docker_dir))
         return props
 
+    def get_mpc_properties(self, deploy_ip: str, node_index: int):
+        props = self.to_properties()
+        # the zone config
+        props.update(self.env_config.to_properties())
+        # the service config
+        props.update(self.mpc_config.to_properties(deploy_ip, node_index))
+        props.update(self.hdfs_storage_config.to_properties())
+        # the config mount info
+        docker_prefix_path = constant.ConfigInfo.wedpr_mpc_docker_dir
+        props.update(
+            {constant.ConfigProperities.WEDPR_CONFIG_DIR: "config.ini"})
+        path = constant.ConfigInfo.get_docker_path(
+            f"{docker_prefix_path}/config.ini")
+        props.update(
+            {constant.ConfigProperities.DOCKER_CONF_PATH: path})
+
+        # specify the log path to mount
+        props.update({constant.ConfigProperities.WEDPR_LOG_DIR: "log"})
+        props.update({constant.ConfigProperities.DOCKER_LOG_PATH:
+                      constant.ConfigInfo.get_docker_path(f"{docker_prefix_path}/log")})
+        return props
+
     def __update_dml__(self, sql, dml_file_path, use_double_quote=False):
         command = "echo '%s' >> %s" % (sql, dml_file_path)
         if use_double_quote:
@@ -1031,8 +1084,8 @@ def __init__(self, config, component_switch: ComponentSwitch):
         self.gateway_sm_ssl = utilities.get_value(
             self.config, crypto_section, "gateway_sm_ssl", False, False)
         # the rpc disable ssl or not
-        self.rpc_disable_ssl = utilities.get_value(
-            self.config, crypto_section, "rpc_disable_ssl", False, False)
+        # self.rpc_disable_ssl = utilities.get_value(
+        #    self.config, crypto_section, "rpc_disable_ssl", False, False)
         # the rpc use sm-ssl or not
         self.rpc_sm_ssl = utilities.get_value(
             self.config, crypto_section, "rpc_sm_ssl", False, False)