Skip to content

refactor(interactive): Unify devcontainer.json of graphscope-dev and flex #173

refactor(interactive): Unify devcontainer.json of graphscope-dev and flex

refactor(interactive): Unify devcontainer.json of graphscope-dev and flex #173

name: Build GraphScope Development and Wheel Images
# build images for:
# 1) wheel: including all dependencies for graphscope's wheel package.
# 2) graphscope-dev: including all dependencies for graphscope's development env.
# 3) vineyard-dev: including all vineyard-related dependencies that could compile graphscope analytical engine.
# 4) vineyard-runtime: including all vineyard-related running dependencies.
# Note that:
# Due to security considerations, we cannot use self-hosts runner(aarch64) when we configured the secret on github.
on:
workflow_dispatch:
inputs:
v6d_version:
description: 'Version for Vineyard (v6d)'
required: true
default: 'main'
build_wheel:
description: 'Whether to build dev-wheel image'
required: true
default: true
type: boolean
build_graphscope_dev:
description: 'Whether to build graphscope-dev image'
required: true
default: true
type: boolean
build_vineyard_dev:
description: 'Whether to build vineyard-dev(runtime) image'
required: true
default: true
type: boolean
pull_request:
branches:
- main
paths:
- 'python/graphscope/gsctl/scripts/**'
concurrency:
group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
cancel-in-progress: true
env:
REGISTRY: registry.cn-hongkong.aliyuncs.com
jobs:
build-wheel-image-x86-64:
runs-on: ubuntu-20.04
if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_wheel == 'true') || (github.event_name == 'pull_request')
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Build Image
run: |
# build wheel image with specified v6d's version
cd ${GITHUB_WORKSPACE}/k8s
VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
if [[ -n ${VINEYARD_VERSION} ]]; then
# graphscope/graphscope-dev:wheel-<version>-x86_64
make dev-wheel VINEYARD_VERSION=${VINEYARD_VERSION}
else
# pull_request: use default vineyard_version
make dev-wheel
fi
- name: Release Image
if: ${{ github.event_name == 'workflow_dispatch' }}
env:
docker_password: ${{ secrets.DOCKER_PASSWORD }}
docker_username: ${{ secrets.DOCKER_USER }}
run: |
echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
# x86_64
arch=$(uname -m)
# image tag
tag=${{ github.event.inputs.v6d_version }}
# dev-wheel image
sudo docker tag graphscope/graphscope-dev:wheel-${tag}-${arch} ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag}
build-wheel-image-aarch64:
runs-on: [self-hosted, Linux, ARM64]
# if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_wheel == 'true') || (github.event_name == 'pull_request')
if: false
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Build Image
run: |
# build wheel image with specified v6d's version
cd ${GITHUB_WORKSPACE}/k8s
VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
if [[ -n ${VINEYARD_VERSION} ]]; then
# graphscope/graphscope-dev:wheel-<version>-aarch64
make dev-wheel VINEYARD_VERSION=${VINEYARD_VERSION}
else
# pull_request: use default vineyard_version
make dev-wheel
fi
- name: Release Image
if: ${{ github.event_name == 'workflow_dispatch' }}
env:
docker_password: ${{ secrets.DOCKER_PASSWORD }}
docker_username: ${{ secrets.DOCKER_USER }}
run: |
echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
# aarch64
arch=$(uname -m)
# image tag
tag=${{ github.event.inputs.v6d_version }}-${arch}
# dev-wheel image
sudo docker tag graphscope/graphscope-dev:wheel-${tag} ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag}
- name: Clean Image
run: |
# aarch64
arch=$(uname -m)
# image tag
tag=${{ github.event.inputs.v6d_version }}-${arch}
# clean
sudo docker rmi -f graphscope/graphscope-dev:wheel-${tag} || true
sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag} || true
manifest-push-wheel-image:
runs-on: ubuntu-20.04
# if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_wheel == 'true' }}
if: false
needs: [build-wheel-image-x86-64, build-wheel-image-aarch64]
steps:
- name: Create and Push Docker Manifest
env:
docker_password: ${{ secrets.DOCKER_PASSWORD }}
docker_username: ${{ secrets.DOCKER_USER }}
run: |
echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
# manifest create
sudo docker manifest create \
${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${{ github.event.inputs.v6d_version }} \
${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${{ github.event.inputs.v6d_version }}-x86_64 \
${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${{ github.event.inputs.v6d_version }}-aarch64
# manifest push
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${{ github.event.inputs.v6d_version }}
build-graphscope-dev-image-x86-64:
runs-on: ubuntu-20.04
if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev == 'true') || (github.event_name == 'pull_request')
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Build Image
run: |
# build graphscope dev image with specified v6d's version
cd ${GITHUB_WORKSPACE}/k8s
VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
if [[ -n ${VINEYARD_VERSION} ]]; then
# graphscope/graphscope-dev:<version>-x86_64
make graphscope-dev VINEYARD_VERSION=${VINEYARD_VERSION}
else
# pull_request: use default vineyard_version
make graphscope-dev
fi
- name: Release Image
if: ${{ github.event_name == 'workflow_dispatch' }}
env:
docker_password: ${{ secrets.DOCKER_PASSWORD }}
docker_username: ${{ secrets.DOCKER_USER }}
run: |
echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
# x86_64
arch=$(uname -m)
# image tag
tag=${{ github.event.inputs.v6d_version }}
# graphscope-dev image
sudo docker tag graphscope/graphscope-dev:${tag}-${arch} ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag}
build-graphscope-dev-image-aarch64:
runs-on: [self-hosted, Linux, ARM64]
# if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev == 'true') || (github.event_name == 'pull_request')
if: false
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Build Image
run: |
# build graphscope dev image with specified v6d's version
cd ${GITHUB_WORKSPACE}/k8s
VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
if [[ -n ${VINEYARD_VERSION} ]]; then
# graphscope/graphscope-dev:<version>-aarch64
make graphscope-dev VINEYARD_VERSION=${VINEYARD_VERSION}
else
# pull_request: use default vineyard_version
make graphscope-dev
fi
- name: Release Image
if: ${{ github.event_name == 'workflow_dispatch' }}
env:
docker_password: ${{ secrets.DOCKER_PASSWORD }}
docker_username: ${{ secrets.DOCKER_USER }}
run: |
echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
# aarch64
arch=$(uname -m)
# image tag
tag=${{ github.event.inputs.v6d_version }}-${arch}
# graphscope-dev image
sudo docker tag graphscope/graphscope-dev:${tag} ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag}
- name: Clean Image
run: |
# aarch64
arch=$(uname -m)
# image tag
tag=${{ github.event.inputs.v6d_version }}-${arch}
# clean
sudo docker rmi -f graphscope/graphscope-dev:${tag} || true
sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag} || true
manifest-push-graphscope-dev-image:
runs-on: ubuntu-20.04
# if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_graphscope_dev == 'true' }}
if: false
needs: [build-graphscope-dev-image-x86-64, build-graphscope-dev-image-aarch64]
steps:
- name: Create and Push Docker Manifest
env:
docker_password: ${{ secrets.DOCKER_PASSWORD }}
docker_username: ${{ secrets.DOCKER_USER }}
run: |
echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
# manifest create
sudo docker manifest create \
${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }} \
${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}-x86_64 \
${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}-aarch64
# manifest push
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}
build-vineyard-dev-image-x86-64:
runs-on: ubuntu-20.04
if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_vineyard_dev == 'true') || (github.event_name == 'pull_request')
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Build Image
run: |
# build vineyard dev image with specified v6d's version
cd ${GITHUB_WORKSPACE}/k8s
VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
if [[ -n ${VINEYARD_VERSION} ]]; then
# graphscope/vineyard-dev:<version>-x86_64
make vineyard-dev VINEYARD_VERSION=${VINEYARD_VERSION}
else
# pull_request: use default vineyard_version
make vineyard-dev
fi
- name: Release Image
if: ${{ github.event_name == 'workflow_dispatch' }}
env:
docker_password: ${{ secrets.DOCKER_PASSWORD }}
docker_username: ${{ secrets.DOCKER_USER }}
run: |
echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
# x86_64
arch=$(uname -m)
# image tag
tag=${{ github.event.inputs.v6d_version }}
# vineyard-dev image
sudo docker tag graphscope/vineyard-dev:${tag}-${arch} ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag}
build-vineyard-dev-image-aarch64:
runs-on: [self-hosted, Linux, ARM64]
# if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_vineyard_dev == 'true') || (github.event_name == 'pull_request')
if: false
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Build Image
run: |
# build vineyard dev image with specified v6d's version
cd ${GITHUB_WORKSPACE}/k8s
VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
if [[ -n ${VINEYARD_VERSION} ]]; then
# graphscope/vineyard-dev:<version>-aarch64
make vineyard-dev VINEYARD_VERSION=${VINEYARD_VERSION}
else
# pull_request: use default vineyard_version
make vineyard-dev
fi
- name: Release Image
if: ${{ github.event_name == 'workflow_dispatch' }}
env:
docker_password: ${{ secrets.DOCKER_PASSWORD }}
docker_username: ${{ secrets.DOCKER_USER }}
run: |
echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
# aarch64
arch=$(uname -m)
# image tag
tag=${{ github.event.inputs.v6d_version }}-${arch}
# vineyard-dev image
sudo docker tag graphscope/vineyard-dev:${tag} ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag}
- name: Clean Image
run: |
# aarch64
arch=$(uname -m)
# image tag
tag=${{ github.event.inputs.v6d_version }}-${arch}
# clean
sudo docker rmi -f graphscope/vineyard-dev:${tag} || true
sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag} || true
manifest-push-vineyard-dev-image:
runs-on: ubuntu-20.04
# if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }}
if: false
needs: [build-vineyard-dev-image-x86-64, build-vineyard-dev-image-aarch64]
steps:
- name: Create and Push Docker Manifest
env:
docker_password: ${{ secrets.DOCKER_PASSWORD }}
docker_username: ${{ secrets.DOCKER_USER }}
run: |
echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
# manifest create
sudo docker manifest create \
${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }} \
${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}-x86_64 \
${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}-aarch64
# manifest push
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}
build-vineyard-runtime-image-x86-64:
runs-on: ubuntu-20.04
# only trigger this step in 'workflow_dispatch' event,
# since the 'vineyard-dev' image isn't actually pushed in 'pull_request'
if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} || (github.event_name == 'pull_request')
needs: [build-vineyard-dev-image-x86-64]
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Build Image
run: |
# build vineyard runtime image with specified v6d's version
cd ${GITHUB_WORKSPACE}/k8s
VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
if [[ -n ${VINEYARD_VERSION} ]]; then
# graphscope/vineyard-runtime:<version>-x86_64
make vineyard-runtime VINEYARD_VERSION=${VINEYARD_VERSION}
else
# pull_request: use default vineyard_version
make vineyard-runtime
fi
- name: Release Image
if: ${{ github.event_name == 'workflow_dispatch' }}
env:
docker_password: ${{ secrets.DOCKER_PASSWORD }}
docker_username: ${{ secrets.DOCKER_USER }}
run: |
echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
# x86_64
arch=$(uname -m)
# image tag
tag=${{ github.event.inputs.v6d_version }}
# vineyard-runtime image
sudo docker tag graphscope/vineyard-runtime:${tag}-${arch} ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag}
build-vineyard-runtime-image-aarch64:
runs-on: [self-hosted, Linux, ARM64]
# only trigger this step in 'workflow_dispatch' event,
# since the 'vineyard-dev' image isn't actually pushed in 'pull_request'
# if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} || (github.event_name == 'pull_request')
if: false
needs: [manifest-push-vineyard-dev-image]
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Build Image
run: |
# build vineyard dev image with specified v6d's version
cd ${GITHUB_WORKSPACE}/k8s
VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
if [[ -n ${VINEYARD_VERSION} ]]; then
# graphscope/vineyard-runtime:<version>-aarch64
make vineyard-runtime VINEYARD_VERSION=${VINEYARD_VERSION}
else
# pull_request: use default vineyard_version
make vineyard-runtime
fi
- name: Release Image
env:
docker_password: ${{ secrets.DOCKER_PASSWORD }}
docker_username: ${{ secrets.DOCKER_USER }}
run: |
echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
# aarch64
arch=$(uname -m)
# image tag
tag=${{ github.event.inputs.v6d_version }}-${arch}
# vineyard-runtime image
sudo docker tag graphscope/vineyard-runtime:${tag} ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag}
- name: Clean Image
run: |
# aarch64
arch=$(uname -m)
# image tag
tag=${{ github.event.inputs.v6d_version }}-${arch}
# clean
sudo docker rmi -f graphscope/vineyard-runtime:${tag} || true
sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag} || true
manifest-push-vineyard-runtime-image:
runs-on: ubuntu-20.04
# if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }}
if: false
needs: [build-vineyard-runtime-image-x86-64, build-vineyard-runtime-image-aarch64]
steps:
- name: Create and Push Docker Manifest
env:
docker_password: ${{ secrets.DOCKER_PASSWORD }}
docker_username: ${{ secrets.DOCKER_USER }}
run: |
echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
# manifest create
sudo docker manifest create \
${{ env.REGISTRY }}/graphscope/vineyard-runtime:${{ github.event.inputs.v6d_version }} \
${{ env.REGISTRY }}/graphscope/vineyard-runtime:${{ github.event.inputs.v6d_version }}-x86_64 \
${{ env.REGISTRY }}/graphscope/vineyard-runtime:${{ github.event.inputs.v6d_version }}-aarch64
# manifest push
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${{ github.event.inputs.v6d_version }}