From 6cc2c8c9f23b0ad82938151a04700fcef1f9d7ad Mon Sep 17 00:00:00 2001 From: Per Halvorsen Date: Sun, 20 Oct 2024 13:26:47 +0200 Subject: [PATCH] add publish make target --- .github/workflows/deploy.yml | 44 +++++++++++++++++++----------------- makefile | 38 +++++++++++++++++++------------ 2 files changed, 47 insertions(+), 35 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index aae9127..beb86f2 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -14,9 +14,11 @@ env: SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }} WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} MODEL_REGISTERY: ${{ secrets.MODEL_REGISTERY }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} jobs: - build-push-model-server: + build-push-images: permissions: contents: 'read' id-token: 'write' @@ -48,34 +50,34 @@ jobs: env: MODEL_REGISTERY: ${{ env.MODEL_REGISTERY }} - build-push-pipeline-worker: - permissions: - contents: 'read' - id-token: 'write' + - name: Build and Push Pipeline Worker + run: make build-push-pipeline-worker + env: + MODEL_REGISTERY: ${{ env.MODEL_REGISTERY }} + + publish-model-server: + name: Push Model Server Image to Docker Hub runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - - name: Google Auth - id: auth - uses: 'google-github-actions/auth@v2' + - name: Docker Auth (Docker Hub) + id: docker-auth-dockerhub + uses: docker/login-action@v3 with: - token_format: 'access_token' - project_id: ${{ env.PROJECT_ID }} - service_account: ${{ env.SERVICE_ACCOUNT }} - workload_identity_provider: ${{ env.WORKLOAD_IDENTITY_PROVIDER }} + username: ${{env.DOCKERHUB_USERNAME}} + password: ${{env.DOCKERHUB_TOKEN}} - - name: Docker Auth - id: docker-auth - uses: 'docker/login-action@v1' - with: - username: 'oauth2accesstoken' - password: '${{ steps.auth.outputs.access_token }}' - registry: '${{ env.MODEL_REGISTERY }}' + - name: Publish Latest Model Server Image + run: make publish-latest-model-server + env: + MODEL_REGISTERY: ${{ env.MODEL_REGISTERY }} + PUBLIC_MODEL_REGISTERY: docker.io/${{ env.DOCKERHUB_USERNAME }} - - name: Build and Push Pipeline Worker - run: make build-push-pipeline-worker + - name: Publish Latest Pipeline Worker Image + run: make publish-latest-pipeline-worker env: MODEL_REGISTERY: ${{ env.MODEL_REGISTERY }} + PUBLIC_MODEL_REGISTERY: docker.io/${{ env.DOCKERHUB_USERNAME }} \ No newline at end of file diff --git a/makefile b/makefile index 6c3db18..4be920b 100644 --- a/makefile +++ b/makefile @@ -4,7 +4,9 @@ TAG := $(VERSION)-$(GIT_SHA) PIPELINE_IMAGE_NAME := whale-speech/pipeline MODEL_SERVER_IMAGE_NAME := whale-speech/model-server PIPELINE_WORKER_IMAGE_NAME := whale-speech/pipeline-worker -ENV_LOCATION := .env + +PUBLIC_MODEL_SERVER_IMAGE_NAME := $(shell echo $(MODEL_SERVER_IMAGE_NAME) | sed 's/\//-/g') +PUBLIC_PIPELINE_WORKER_IMAGE_NAME := $(shell echo $(PIPELINE_WORKER_IMAGE_NAME) | sed 's/\//-/g') local-run: bash scripts/kill_model_server.sh @@ -27,20 +29,15 @@ gcp-deduplicate: python3 src/gcp.py --deduplicate setup: - sudo apt-get update - sudo apt-get install python3-venv libhdf5-dev libsndfile1 gcc - python3 -m venv $(ENV_LOCATION) - $(ENV_LOCATION)/bin/pip install -r requirements/requirements.txt - -run: - $(ENV_LOCATION)/bin/python3 src/pipeline.py - -# Used by GHA -install: + conda create -n whale-speech python=3.11 + conda activate whale-speech sudo apt-get update sudo apt-get install libhdf5-dev libsndfile1 gcc python3 -m pip install -r requirements/requirements.txt + python3 -m pip install -r requirements/model-requirements.txt + +# Docker related check-uncommited: git diff-index --quiet HEAD @@ -53,6 +50,8 @@ push: check-uncommited build-push: build push + +# Model server related build-model-server: check-uncommited docker build -t $(MODEL_SERVER_IMAGE_NAME):$(TAG) --platform linux/amd64 -f Dockerfile.model-server . @@ -64,9 +63,17 @@ push-model-server-latest: check-uncommited docker tag $(MODEL_SERVER_IMAGE_NAME):$(TAG) $(MODEL_REGISTERY)/$(MODEL_SERVER_IMAGE_NAME):latest docker push $(MODEL_REGISTERY)/$(MODEL_SERVER_IMAGE_NAME):latest -# Used by GHA build-push-model-server: build-model-server push-model-server push-model-server-latest +publish-latest-model-server: build-model-server + docker tag $(MODEL_SERVER_IMAGE_NAME):$(TAG) $(PUBLIC_MODEL_REGISTERY)/$(PUBLIC_MODEL_SERVER_IMAGE_NAME):latest + docker push $(PUBLIC_MODEL_REGISTERY)/$(PUBLIC_MODEL_SERVER_IMAGE_NAME):latest + +test-server: + python3 examples/test_server.py + + +# Pipeline worker related build-pipeline-worker: check-uncommited docker build -t $(PIPELINE_WORKER_IMAGE_NAME):$(TAG) --platform linux/amd64 -f Dockerfile.pipeline-worker . @@ -80,9 +87,12 @@ push-pipeline-worker-latest: check-uncommited build-push-pipeline-worker: build-pipeline-worker push-pipeline-worker push-pipeline-worker-latest -test-server: - python3 examples/test_server.py +publish-latest-pipeline-worker: build-pipeline-worker + docker tag $(PIPELINE_WORKER_IMAGE_NAME):$(TAG) $(PUBLIC_MODEL_REGISTERY)/$(PUBLIC_PIPELINE_WORKER_IMAGE_NAME):latest + docker push $(PUBLIC_MODEL_REGISTERY)/$(PUBLIC_PIPELINE_WORKER_IMAGE_NAME):latest + +# Pipeline run related run-dataflow: python3 src/pipeline.py \ --job_name "whale-speech-$(GIT_SHA)" \