add publish make target

pmhalvor · Oct 20, 2024 · 6cc2c8c · 6cc2c8c
1 parent 8cd65f6
commit 6cc2c8c
Show file tree

Hide file tree

Showing 2 changed files with 47 additions and 35 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -14,9 +14,11 @@ env:
   SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
   WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
   MODEL_REGISTERY: ${{ secrets.MODEL_REGISTERY }}
+  DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+  DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
 jobs:
-  build-push-model-server:
+  build-push-images:
     permissions:
       contents: 'read'
       id-token: 'write'
@@ -48,34 +50,34 @@ jobs:
         env:
           MODEL_REGISTERY: ${{ env.MODEL_REGISTERY }}
 
-  build-push-pipeline-worker:
-    permissions:
-      contents: 'read'
-      id-token: 'write'
+      - name: Build and Push Pipeline Worker
+        run: make build-push-pipeline-worker
+        env:
+          MODEL_REGISTERY: ${{ env.MODEL_REGISTERY }}
 
+
+  publish-model-server:
+    name: Push Model Server Image to Docker Hub
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Google Auth
-        id: auth
-        uses: 'google-github-actions/auth@v2'
+      - name: Docker Auth (Docker Hub)
+        id: docker-auth-dockerhub
+        uses: docker/login-action@v3
         with:
-          token_format: 'access_token'
-          project_id: ${{ env.PROJECT_ID }}
-          service_account: ${{ env.SERVICE_ACCOUNT }}
-          workload_identity_provider: ${{ env.WORKLOAD_IDENTITY_PROVIDER }}
+          username: ${{env.DOCKERHUB_USERNAME}}
+          password: ${{env.DOCKERHUB_TOKEN}}
 
-      - name: Docker Auth
-        id: docker-auth
-        uses: 'docker/login-action@v1'
-        with:
-          username: 'oauth2accesstoken'
-          password: '${{ steps.auth.outputs.access_token }}'
-          registry: '${{ env.MODEL_REGISTERY }}'
+      - name: Publish Latest Model Server Image
+        run: make publish-latest-model-server
+        env:
+          MODEL_REGISTERY: ${{ env.MODEL_REGISTERY }}
+          PUBLIC_MODEL_REGISTERY: docker.io/${{ env.DOCKERHUB_USERNAME }}
 
-      - name: Build and Push Pipeline Worker
-        run: make build-push-pipeline-worker
+      - name: Publish Latest Pipeline Worker Image
+        run: make publish-latest-pipeline-worker
         env:
           MODEL_REGISTERY: ${{ env.MODEL_REGISTERY }}
+          PUBLIC_MODEL_REGISTERY: docker.io/${{ env.DOCKERHUB_USERNAME }}
diff --git a/makefile b/makefile
@@ -4,7 +4,9 @@ TAG := $(VERSION)-$(GIT_SHA)
 PIPELINE_IMAGE_NAME := whale-speech/pipeline
 MODEL_SERVER_IMAGE_NAME := whale-speech/model-server
 PIPELINE_WORKER_IMAGE_NAME := whale-speech/pipeline-worker
-ENV_LOCATION := .env
+
+PUBLIC_MODEL_SERVER_IMAGE_NAME := $(shell echo $(MODEL_SERVER_IMAGE_NAME) | sed 's/\//-/g')
+PUBLIC_PIPELINE_WORKER_IMAGE_NAME := $(shell echo $(PIPELINE_WORKER_IMAGE_NAME) | sed 's/\//-/g')
 
 local-run: 
 	bash scripts/kill_model_server.sh
@@ -27,20 +29,15 @@ gcp-deduplicate:
 	python3 src/gcp.py --deduplicate
 
 setup:
-	sudo apt-get update
-	sudo apt-get install python3-venv libhdf5-dev libsndfile1 gcc
-	python3 -m venv $(ENV_LOCATION)
-	$(ENV_LOCATION)/bin/pip install -r requirements/requirements.txt
-
-run: 
-	$(ENV_LOCATION)/bin/python3 src/pipeline.py
-
-# Used by GHA
-install:
+	conda create -n whale-speech python=3.11
+	conda activate whale-speech
 	sudo apt-get update
 	sudo apt-get install libhdf5-dev libsndfile1 gcc
 	python3 -m pip install -r requirements/requirements.txt
+	python3 -m pip install -r requirements/model-requirements.txt
 
+
+# Docker related
 check-uncommited:
 	git diff-index --quiet HEAD
 
@@ -53,6 +50,8 @@ push: check-uncommited
 
 build-push: build push
 
+
+# Model server related
 build-model-server: check-uncommited
 	docker build -t $(MODEL_SERVER_IMAGE_NAME):$(TAG) --platform linux/amd64 -f Dockerfile.model-server .
 
@@ -64,9 +63,17 @@ push-model-server-latest: check-uncommited
 	docker tag $(MODEL_SERVER_IMAGE_NAME):$(TAG) $(MODEL_REGISTERY)/$(MODEL_SERVER_IMAGE_NAME):latest
 	docker push $(MODEL_REGISTERY)/$(MODEL_SERVER_IMAGE_NAME):latest
 
-# Used by GHA
 build-push-model-server: build-model-server push-model-server push-model-server-latest
 
+publish-latest-model-server: build-model-server
+	docker tag $(MODEL_SERVER_IMAGE_NAME):$(TAG) $(PUBLIC_MODEL_REGISTERY)/$(PUBLIC_MODEL_SERVER_IMAGE_NAME):latest
+	docker push $(PUBLIC_MODEL_REGISTERY)/$(PUBLIC_MODEL_SERVER_IMAGE_NAME):latest
+
+test-server:
+	python3 examples/test_server.py
+
+
+# Pipeline worker related
 build-pipeline-worker: check-uncommited
 	docker build -t $(PIPELINE_WORKER_IMAGE_NAME):$(TAG) --platform linux/amd64 -f Dockerfile.pipeline-worker .
 
@@ -80,9 +87,12 @@ push-pipeline-worker-latest: check-uncommited
 
 build-push-pipeline-worker: build-pipeline-worker push-pipeline-worker push-pipeline-worker-latest
 
-test-server:
-	python3 examples/test_server.py
+publish-latest-pipeline-worker: build-pipeline-worker
+	docker tag $(PIPELINE_WORKER_IMAGE_NAME):$(TAG) $(PUBLIC_MODEL_REGISTERY)/$(PUBLIC_PIPELINE_WORKER_IMAGE_NAME):latest
+	docker push $(PUBLIC_MODEL_REGISTERY)/$(PUBLIC_PIPELINE_WORKER_IMAGE_NAME):latest
+
 
+# Pipeline run related
 run-dataflow:
 	python3 src/pipeline.py \
 		--job_name "whale-speech-$(GIT_SHA)" \