add similar job for pipeline worker

pmhalvor · Oct 20, 2024 · 4886b5d · 4886b5d
1 parent b9263af
commit 4886b5d
Show file tree

Hide file tree

Showing 2 changed files with 60 additions and 18 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -4,10 +4,11 @@ on:
   push:
     branches:
       - main  # Runs on push to the main branch
-  pull_request:
+  pull_request: # TODO remove, so only merges to main trigger this 
     branches:
       - main  # Runs on pull requests to the main branch
 
+
 env:
   PROJECT_ID: ${{ secrets.PROJECT_ID }}
   SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
@@ -42,7 +43,39 @@ jobs:
           password: '${{ steps.auth.outputs.access_token }}'
           registry: '${{ env.MODEL_REGISTERY }}'
 
-      - name: Build and Push Model Server
-        run: make build-push-model-server
+      - name: Build and Push Model Server (latest)
+        run: make build-push-model-server-latest
+        env:
+          MODEL_REGISTERY: ${{ env.MODEL_REGISTERY }}
+
+  build-push-pipeline-worker:
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Google Auth
+        id: auth
+        uses: 'google-github-actions/auth@v2'
+        with:
+          token_format: 'access_token'
+          project_id: ${{ env.PROJECT_ID }}
+          service_account: ${{ env.SERVICE_ACCOUNT }}
+          workload_identity_provider: ${{ env.WORKLOAD_IDENTITY_PROVIDER }}
+
+      - name: Docker Auth
+        id: docker-auth
+        uses: 'docker/login-action@v1'
+        with:
+          username: 'oauth2accesstoken'
+          password: '${{ steps.auth.outputs.access_token }}'
+          registry: '${{ env.MODEL_REGISTERY }}'
+
+      - name: Build and Push Pipeline Worker (latest)
+        run: make build-push-pipeline-worker-latest
         env:
           MODEL_REGISTERY: ${{ env.MODEL_REGISTERY }}
diff --git a/makefile b/makefile
@@ -1,8 +1,9 @@
 VERSION := 0.0.0
 GIT_SHA := $(shell git rev-parse --short HEAD)
-PIPELINE_IMAGE_NAME := whale-speech/pipeline:$(VERSION)-$(GIT_SHA)
-MODEL_SERVER_IMAGE_NAME := whale-speech/model-server:$(VERSION)-$(GIT_SHA)
-PIPELINE_WORKER_IMAGE_NAME := whale-speech/pipeline-worker:$(VERSION)-$(GIT_SHA)
+TAG := $(VERSION)-$(GIT_SHA)
+PIPELINE_IMAGE_NAME := whale-speech/pipeline
+MODEL_SERVER_IMAGE_NAME := whale-speech/model-server
+PIPELINE_WORKER_IMAGE_NAME := whale-speech/pipeline-worker
 ENV_LOCATION := .env
 
 local-run: 
@@ -44,32 +45,40 @@ check-uncommited:
 	git diff-index --quiet HEAD
 
 build: check-uncommited
-	docker build -t $(PIPELINE_IMAGE_NAME) --platform linux/amd64 .
+	docker build -t $(PIPELINE_IMAGE_NAME):$(TAG) --platform linux/amd64 .
 
 push: check-uncommited
-	docker tag $(PIPELINE_IMAGE_NAME) $(MODEL_REGISTERY)/$(PIPELINE_IMAGE_NAME)
-	docker push $(MODEL_REGISTERY)/$(PIPELINE_IMAGE_NAME)
+	docker tag $(PIPELINE_IMAGE_NAME):$(TAG) $(MODEL_REGISTERY)/$(PIPELINE_IMAGE_NAME):$(TAG)
+	docker push $(MODEL_REGISTERY)/$(PIPELINE_IMAGE_NAME):$(TAG)
 
 build-push: build push
 
 build-model-server: check-uncommited
-	docker build -t $(MODEL_SERVER_IMAGE_NAME) --platform linux/amd64 -f Dockerfile.model-server .
+	docker build -t $(MODEL_SERVER_IMAGE_NAME):$(TAG) --platform linux/amd64 -f Dockerfile.model-server .
 
 push-model-server: check-uncommited
-	docker tag $(MODEL_SERVER_IMAGE_NAME) $(MODEL_REGISTERY)/$(MODEL_SERVER_IMAGE_NAME)
-	docker push $(MODEL_REGISTERY)/$(MODEL_SERVER_IMAGE_NAME)
+	docker tag $(MODEL_SERVER_IMAGE_NAME):$(TAG) $(MODEL_REGISTERY)/$(MODEL_SERVER_IMAGE_NAME):$(TAG)
+	docker push $(MODEL_REGISTERY)/$(MODEL_SERVER_IMAGE_NAME):$(TAG)
+
+push-model-server-latest: check-uncommited
+	docker tag $(MODEL_SERVER_IMAGE_NAME):$(TAG) $(MODEL_REGISTERY)/$(MODEL_SERVER_IMAGE_NAME):latest
+	docker push $(MODEL_REGISTERY)/$(MODEL_SERVER_IMAGE_NAME):latest
 
 # Used by GHA
-build-push-model-server: build-model-server push-model-server
+build-push-model-server: build-model-server push-model-server push-model-server-latest
 
 build-pipeline-worker: check-uncommited
-	docker build -t $(PIPELINE_WORKER_IMAGE_NAME) --platform linux/amd64 -f Dockerfile.pipeline-worker .
+	docker build -t $(PIPELINE_WORKER_IMAGE_NAME):$(TAG) --platform linux/amd64 -f Dockerfile.pipeline-worker .
 
 push-pipeline-worker: check-uncommited
-	docker tag $(PIPELINE_WORKER_IMAGE_NAME) $(MODEL_REGISTERY)/$(PIPELINE_WORKER_IMAGE_NAME)
-	docker push $(MODEL_REGISTERY)/$(PIPELINE_WORKER_IMAGE_NAME)
+	docker tag $(PIPELINE_WORKER_IMAGE_NAME):$(TAG) $(MODEL_REGISTERY)/$(PIPELINE_WORKER_IMAGE_NAME):$(TAG)
+	docker push $(MODEL_REGISTERY)/$(PIPELINE_WORKER_IMAGE_NAME):$(TAG)
+
+push-pipeline-worker-latest: check-uncommited
+	docker tag $(PIPELINE_WORKER_IMAGE_NAME):$(TAG) $(MODEL_REGISTERY)/$(PIPELINE_WORKER_IMAGE_NAME):latest
+	docker push $(MODEL_REGISTERY)/$(PIPELINE_WORKER_IMAGE_NAME):latest
 
-build-push-pipeline-worker: build-pipeline-worker push-pipeline-worker
+build-push-pipeline-worker-latest: build-pipeline-worker push-pipeline-worker push-pipeline-worker-latest
 
 test-server:
 	python3 examples/test_server.py
@@ -86,7 +95,7 @@ run-dataflow:
 		--num_workers=8 \
 		--max_num_workers=8 \
 		--autoscaling_algorithm=THROUGHPUT_BASED \
-		--worker_harness_container_image=$(MODEL_REGISTERY)/$(PIPELINE_WORKER_IMAGE_NAME) \
+		--worker_harness_container_image=$(MODEL_REGISTERY)/$(PIPELINE_WORKER_IMAGE_NAME):$(TAG) \
 		--start "2024-07-11" \
 		--end "2024-07-11" \
 		--offset 0 \