From 6850282d40e64e6caa93c9296e5078d2b942e9b9 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Sun, 8 Dec 2024 20:37:49 +0800 Subject: [PATCH 1/2] fix docker file --- .github/workflows/docker.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 147d6d9d..36dd2053 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -64,17 +64,18 @@ jobs: [[ ${{github.ref}} == */tags/* ]] && PIR_DOCKER_TAG="${GIT_TAG_NAME}" || PIR_DOCKER_TAG="${BRANCH_NAME}" PIR_DOCKER_TAG="fiscoorg/wedpr-pir:${PIR_DOCKER_TAG}" echo "* Begin to build wedpr-pir docker: ${PIR_DOCKER_TAG}" - cd WeDPR-Component/docker-files/cpp/ + cd WeDPR/docker-files/site/ docker build --target wedpr-pir -t ${PIR_DOCKER_TAG} . echo "* Build wedpr-pir docker: ${PIR_DOCKER_TAG} success" docker push ${PIR_DOCKER_TAG} echo "* Push wedpr-pir docker: ${PIR_DOCKER_TAG} success" # the wedpr-site image - name: Build and Push wedpr-site images + run: | [[ ${{github.ref}} == */tags/* ]] && SITE_DOCKER_TAG="${GIT_TAG_NAME}" || SITE_DOCKER_TAG="${BRANCH_NAME}" SITE_DOCKER_TAG="fiscoorg/wedpr-site:${SITE_DOCKER_TAG}" echo "* Begin to build wedpr-site docker: ${SITE_DOCKER_TAG}" - cd WeDPR-Component/docker-files/cpp/ + cd WeDPR/docker-files/site/ docker build --target wedpr-site -t ${SITE_DOCKER_TAG} . echo "* Build wedpr-site docker: ${SITE_DOCKER_TAG} success" docker push ${SITE_DOCKER_TAG} From 6e0a00a6a431643b8da09aedfc6d41056c00e22f Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Sun, 8 Dec 2024 20:42:45 +0800 Subject: [PATCH 2/2] test docker --- .github/workflows/docker.yml | 20 ++++--- .../workflows/docker_jupyter_base_image.yml | 6 +-- .github/workflows/docker_wedpr_image.yml | 6 +-- ChangeLog.md | 52 +++++++++++++++++++ build.gradle | 2 +- docker-files/base/Dockerfile | 2 +- docker-files/jupyter/Dockerfile | 8 +-- docker-files/site/Dockerfile | 8 +-- wedpr-builder/conf/config-example.toml | 14 ++--- 9 files changed, 84 insertions(+), 34 deletions(-) create mode 100644 ChangeLog.md diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 36dd2053..971ab629 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,11 +7,9 @@ on: - "README.md" tags: - 'v3.*.*' - pull_request: - paths-ignore: - - "docs/**" - - "Changelog.md" - - "README.md" + branches: + - main + - feature-milestone2 release: types: [prereleased] @@ -53,8 +51,8 @@ jobs: [[ ${{github.ref}} == */tags/* ]] && JUPYTER_DOCKER_TAG="${GIT_TAG_NAME}" || JUPYTER_DOCKER_TAG="${BRANCH_NAME}" JUPYTER_DOCKER_TAG="fiscoorg/wedpr-jupyter-worker:${JUPYTER_DOCKER_TAG}" echo "* Begin to build wedpr-jupyter-worker docker: ${JUPYTER_DOCKER_TAG}" - cd WeDPR/docker-files/site - docker build --target wedpr-jupyter-worker -t ${JUPYTER_DOCKER_TAG} . + cd docker-files/site + docker build --target wedpr-jupyter-worker --build-arg SOURCE_BRANCH=${BRANCH_NAME} -t ${JUPYTER_DOCKER_TAG} . echo "* Build wedpr-jupyter-worker docker: ${JUPYTER_DOCKER_TAG} success" docker push ${JUPYTER_DOCKER_TAG} echo "* Push wedpr-jupyter-worker docker: ${JUPYTER_DOCKER_TAG} success" @@ -64,8 +62,8 @@ jobs: [[ ${{github.ref}} == */tags/* ]] && PIR_DOCKER_TAG="${GIT_TAG_NAME}" || PIR_DOCKER_TAG="${BRANCH_NAME}" PIR_DOCKER_TAG="fiscoorg/wedpr-pir:${PIR_DOCKER_TAG}" echo "* Begin to build wedpr-pir docker: ${PIR_DOCKER_TAG}" - cd WeDPR/docker-files/site/ - docker build --target wedpr-pir -t ${PIR_DOCKER_TAG} . + cd docker-files/site/ + docker build --target wedpr-pir --build-arg SOURCE_BRANCH=${BRANCH_NAME} -t ${PIR_DOCKER_TAG} . echo "* Build wedpr-pir docker: ${PIR_DOCKER_TAG} success" docker push ${PIR_DOCKER_TAG} echo "* Push wedpr-pir docker: ${PIR_DOCKER_TAG} success" @@ -75,8 +73,8 @@ jobs: [[ ${{github.ref}} == */tags/* ]] && SITE_DOCKER_TAG="${GIT_TAG_NAME}" || SITE_DOCKER_TAG="${BRANCH_NAME}" SITE_DOCKER_TAG="fiscoorg/wedpr-site:${SITE_DOCKER_TAG}" echo "* Begin to build wedpr-site docker: ${SITE_DOCKER_TAG}" - cd WeDPR/docker-files/site/ - docker build --target wedpr-site -t ${SITE_DOCKER_TAG} . + cd docker-files/site/ + docker build --target wedpr-site --build-arg SOURCE_BRANCH=${BRANCH_NAME} -t ${SITE_DOCKER_TAG} . echo "* Build wedpr-site docker: ${SITE_DOCKER_TAG} success" docker push ${SITE_DOCKER_TAG} echo "* Push wedpr-sitedocker: ${SITE_DOCKER_TAG} success" \ No newline at end of file diff --git a/.github/workflows/docker_jupyter_base_image.yml b/.github/workflows/docker_jupyter_base_image.yml index f1b380fc..1f149e90 100644 --- a/.github/workflows/docker_jupyter_base_image.yml +++ b/.github/workflows/docker_jupyter_base_image.yml @@ -31,10 +31,10 @@ jobs: - name: Build and Push wedpr-jupyter-image images run: | echo "* Manual build wedpr-jupyter-image image, docker version: ${{ github.event.inputs.tags }}" - DOCKER_TAG="wedpr-jupyter-image:${{ github.event.inputs.tags }}" + DOCKER_TAG="fiscoorg/wedpr-jupyter-image:${{ github.event.inputs.tags }}" echo "* Begin to build ${DOCKER_TAG}" - cd WeDPR/docker-files/base - docker build docker build -t ${DOCKER_TAG} . + cd docker-files/base + docker build -t ${DOCKER_TAG} . echo "* Build ${DOCKER_TAG} success" docker push ${DOCKER_TAG} echo "* Push ${DOCKER_TAG} success" \ No newline at end of file diff --git a/.github/workflows/docker_wedpr_image.yml b/.github/workflows/docker_wedpr_image.yml index f35cbe87..36b0eaa8 100644 --- a/.github/workflows/docker_wedpr_image.yml +++ b/.github/workflows/docker_wedpr_image.yml @@ -31,10 +31,10 @@ jobs: - name: Build and Push wedpr-image images run: | echo "* Manual build wedpr-image image, docker version: ${{ github.event.inputs.tags }}" - DOCKER_TAG="wedpr-image:${{ github.event.inputs.tags }}" + DOCKER_TAG="fiscoorg/wedpr-image:${{ github.event.inputs.tags }}" echo "* Begin to build ${DOCKER_TAG}" - cd WeDPR/docker-files/base - docker build docker build -t ${DOCKER_TAG} . + cd docker-files/base + docker build -t ${DOCKER_TAG} . echo "* Build ${DOCKER_TAG} success" docker push ${DOCKER_TAG} echo "* Push ${DOCKER_TAG} success" \ No newline at end of file diff --git a/ChangeLog.md b/ChangeLog.md new file mode 100644 index 00000000..3dfb556d --- /dev/null +++ b/ChangeLog.md @@ -0,0 +1,52 @@ +# v3.0.0 +(2024-12-09) + +### 新增 + +WeDPR隐私计算平台站点端管理功能，包括： + +**丰富的隐私计算任务支持** +- 多方(>=2)隐私求交集 +- 匿踪查询任务 +- 多方(>=2)联合建模任务(SecureBoost, SecureLR) +- 多方(>2)数据联合分析任务(隐私SQL, 隐私Python) + +**基于区块链灵活可扩展的多方同步模块** + +- 基于区块链进行跨机构元数据信息同步 + +**丰富的数据源管理** + +- 支持HDFS, Hive, MYSQL和国产数据库等多种数据源接入 + +**细粒度的用户和权限体系** + +- 支持多用户模式，并支持用户维度的数据、服务权限管理 + +**通用的数据、服务审批流** + +- 实现了审批模块，支持数据、服务授权 + + +**基于Jupyter和wedpr-ml-toolkit sdk的专家模式** + +在向导模式的基础上，为数据和模型开发人员提供了基于Jupyter的专家模式，便于其获取建模、隐私求交集的结果，完成后续数据分析工作。 + +- 实现了用户维度的Jupyter管理，多用户的Jupyter环境通过linux用户体系完全隔离开 +- 提供了wedpr-ml-toolkit工具，便于用户在Jupyter专家模式环境中发起隐私求交集、联合建模等隐私计算任务，并可灵活地获取任务执行结果 + + +**支持DAG的任务调度模块** + +- 任务调度模块支持DAG工作流 +- 各类隐私计算节点多活可扩展 + + +**服务发布功能** + +- 数据集可发布为匿踪查询服务，供授权的用户或机构查询 +- 联合建模训练过程中产生的模型可发布为服务，供联合预测使用 + +**API接入** + +- 支持应用方通过申请的凭证AccessKey接入到管理平台 diff --git a/build.gradle b/build.gradle index ce635805..41cc6e88 100644 --- a/build.gradle +++ b/build.gradle @@ -99,7 +99,7 @@ ext { // integrationTest.mustRunAfter test allprojects { group = 'com.webank.wedpr' - version = '1.0.0-rc1-SNAPSHOT' + version = '3.0.0-SNAPSHOT' apply plugin: 'maven-publish' apply plugin: 'com.github.sherter.google-java-format' apply plugin: 'idea' diff --git a/docker-files/base/Dockerfile b/docker-files/base/Dockerfile index bb258491..bac615c0 100644 --- a/docker-files/base/Dockerfile +++ b/docker-files/base/Dockerfile @@ -4,7 +4,7 @@ FROM ubuntu:18.04 as wedpr-image RUN apt-get update && apt-get install -y tzdata language-pack-zh* \ && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo 'Asia/Shanghai' >/etc/timezone \ && locale-gen zh_CN.UTF-8 \ - && apt-get install -y git wget fontconfig tar unzip lzip vim nasm python3.8 python3-pip python3.8-dev perl graphviz openjdk-8-jdk gcc g++ build-essential m4 tcpdump net-tools gdb libkrb5-dev krb5-user pkg-config default-libmysqlclient-dev\ + && apt-get install -y git wget fontconfig tar unzip lzip vim nasm python3.8 python3-pip python3.8-dev perl graphviz openjdk-8-jdk gcc g++ build-essential m4 tcpdump net-tools gdb libkrb5-dev pkg-config default-libmysqlclient-dev\ && rm -f /usr/bin/python && ln -s /usr/bin/python3.8 /usr/bin/python \ && rm -f /usr/bin/python3 && ln -s /usr/bin/python3.8 /usr/bin/python3 \ && ln -s /usr/bin/pip3 /usr/bin/pip diff --git a/docker-files/jupyter/Dockerfile b/docker-files/jupyter/Dockerfile index 0ffe6c04..3312c471 100644 --- a/docker-files/jupyter/Dockerfile +++ b/docker-files/jupyter/Dockerfile @@ -1,4 +1,4 @@ -FROM wedpr-image:latest as wedpr-jupyter-image +FROM ficoorg/wedpr-image:v3.0.0 as wedpr-jupyter-image ENV LANG zh_CN.UTF-8 ENV LANGUAGE zh_CN.UTF-8 @@ -13,7 +13,7 @@ RUN chmod -R 777 /home/share/ # install jupyter RUN pip install --no-cache-dir -i https://pypi.mirrors.ustc.edu.cn/simple/ -r root/requirements.txt -# TODO: use the non-test pip after uploaded -RUN pip install --no-cache-dir -i https://test.pypi.org/simple/ wedpr-authenticator -RUN pip install --no-cache-dir -i https://test.pypi.org/simple/ wedpr-ml-toolkit +# TODO: use the non-test pip after uploaded +RUN pip install --no-cache-dir -i https://test.pypi.org/simple/ wedpr-authenticator==1.0.0.dev-20241124 +RUN pip install --no-cache-dir -i https://test.pypi.org/simple/ wedpr-ml-toolkit==1.0.0.dev-20241201 \ No newline at end of file diff --git a/docker-files/site/Dockerfile b/docker-files/site/Dockerfile index ecef5604..174a38ab 100644 --- a/docker-files/site/Dockerfile +++ b/docker-files/site/Dockerfile @@ -1,5 +1,5 @@ # the builder -FROM wedpr-jupyter-image:latest as builder +FROM fiscoorg/wedpr-jupyter-image:v3.0.0 as builder LABEL maintainer service@webank.com @@ -13,7 +13,7 @@ RUN git clone https://github.com/WeBankBlockchain/WeDPR.git -b ${SOURCE} \ && cd WeDPR && bash gradlew clean build # the wedpr-worker -FROM wedpr-jupyter-image:latest as wedpr-jupyter-worker +FROM fiscoorg/wedpr-jupyter-image:v3.0.0 as wedpr-jupyter-worker LABEL maintainer service@webank.com RUN mkdir -p /data/home/wedpr @@ -21,7 +21,7 @@ COPY --from=builder /WeDPR/wedpr-worker/dist/ /data/home/wedpr/wedpr-worker ENTRYPOINT ["/bin/bash", "/data/home/wedpr/wedpr-worker/start.sh", "true"] # the wedpr-pir -FROM wedpr-image:latest as wedpr-pir +FROM fiscoorg/wedpr-image:v3.0.0 as wedpr-pir LABEL maintainer service@webank.com RUN mkdir -p /data/home/wedpr @@ -29,7 +29,7 @@ COPY --from=builder /WeDPR/wedpr-pir/dist/ /data/app/wedpr-pir ENTRYPOINT ["/bin/bash", "/data/home/wedpr/wedpr-pir/start.sh", "true"] # the wedpr-site -FROM wedpr-image:latest as wedpr-site +FROM fiscoorg/wedpr-image:v3.0.0 as wedpr-site LABEL maintainer service@webank.com # install the requirements diff --git a/wedpr-builder/conf/config-example.toml b/wedpr-builder/conf/config-example.toml index 93bcfa4b..1223f7ee 100644 --- a/wedpr-builder/conf/config-example.toml +++ b/wedpr-builder/conf/config-example.toml @@ -14,13 +14,13 @@ wedpr_model_source_path = "/data/home/wedpr/WeDPR-Component/python/" docker_mode = true #### define the docker images desc ### -wedpr_gateway_service_image_desc = "wedpr-gateway-service:latest" -wedpr_node_service_image_desc = "wedpr-pro-node-service:latest" -wedpr_mpc_service_image_desc = "wedpr-mpc-service:latest" -wedpr_jupyter_worker_image_desc = "wedpr-jupyter-worker:latest" -wedpr_model_image_desc = "wedpr-model-service:latest" -wedpr_site_image_desc = "wedpr-site:latest" -wedpr_pir_image_desc = "wedpr-pir:latest" +wedpr_gateway_service_image_desc = "fiscoorg/wedpr-gateway-service:latest" +wedpr_node_service_image_desc = "fiscoorg/wedpr-pro-node-service:latest" +wedpr_mpc_service_image_desc = "fiscoorg/wedpr-mpc-service:latest" +wedpr_jupyter_worker_image_desc = "fiscoorg/wedpr-jupyter-worker:latest" +wedpr_model_image_desc = "fiscoorg/wedpr-model-service:latest" +wedpr_site_image_desc = "fiscoorg/wedpr-site:latest" +wedpr_pir_image_desc = "fiscoorg/wedpr-pir:latest" deploy_dir = "wedpr-example"