Skip to content

Commit

Permalink
add similar job for pipeline worker
Browse files Browse the repository at this point in the history
pmhalvor committed Oct 20, 2024
1 parent b9263af commit 8cd65f6
Showing 2 changed files with 58 additions and 16 deletions.
35 changes: 34 additions & 1 deletion .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
@@ -4,10 +4,11 @@ on:
push:
branches:
- main # Runs on push to the main branch
pull_request:
pull_request: # TODO remove, so only merges to main trigger this
branches:
- main # Runs on pull requests to the main branch


env:
PROJECT_ID: ${{ secrets.PROJECT_ID }}
SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
@@ -46,3 +47,35 @@ jobs:
run: make build-push-model-server
env:
MODEL_REGISTERY: ${{ env.MODEL_REGISTERY }}

build-push-pipeline-worker:
permissions:
contents: 'read'
id-token: 'write'

runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Google Auth
id: auth
uses: 'google-github-actions/auth@v2'
with:
token_format: 'access_token'
project_id: ${{ env.PROJECT_ID }}
service_account: ${{ env.SERVICE_ACCOUNT }}
workload_identity_provider: ${{ env.WORKLOAD_IDENTITY_PROVIDER }}

- name: Docker Auth
id: docker-auth
uses: 'docker/login-action@v1'
with:
username: 'oauth2accesstoken'
password: '${{ steps.auth.outputs.access_token }}'
registry: '${{ env.MODEL_REGISTERY }}'

- name: Build and Push Pipeline Worker
run: make build-push-pipeline-worker
env:
MODEL_REGISTERY: ${{ env.MODEL_REGISTERY }}
39 changes: 24 additions & 15 deletions makefile
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
VERSION := 0.0.0
GIT_SHA := $(shell git rev-parse --short HEAD)
PIPELINE_IMAGE_NAME := whale-speech/pipeline:$(VERSION)-$(GIT_SHA)
MODEL_SERVER_IMAGE_NAME := whale-speech/model-server:$(VERSION)-$(GIT_SHA)
PIPELINE_WORKER_IMAGE_NAME := whale-speech/pipeline-worker:$(VERSION)-$(GIT_SHA)
TAG := $(VERSION)-$(GIT_SHA)
PIPELINE_IMAGE_NAME := whale-speech/pipeline
MODEL_SERVER_IMAGE_NAME := whale-speech/model-server
PIPELINE_WORKER_IMAGE_NAME := whale-speech/pipeline-worker
ENV_LOCATION := .env

local-run:
@@ -44,32 +45,40 @@ check-uncommited:
git diff-index --quiet HEAD

build: check-uncommited
docker build -t $(PIPELINE_IMAGE_NAME) --platform linux/amd64 .
docker build -t $(PIPELINE_IMAGE_NAME):$(TAG) --platform linux/amd64 .

push: check-uncommited
docker tag $(PIPELINE_IMAGE_NAME) $(MODEL_REGISTERY)/$(PIPELINE_IMAGE_NAME)
docker push $(MODEL_REGISTERY)/$(PIPELINE_IMAGE_NAME)
docker tag $(PIPELINE_IMAGE_NAME):$(TAG) $(MODEL_REGISTERY)/$(PIPELINE_IMAGE_NAME):$(TAG)
docker push $(MODEL_REGISTERY)/$(PIPELINE_IMAGE_NAME):$(TAG)

build-push: build push

build-model-server: check-uncommited
docker build -t $(MODEL_SERVER_IMAGE_NAME) --platform linux/amd64 -f Dockerfile.model-server .
docker build -t $(MODEL_SERVER_IMAGE_NAME):$(TAG) --platform linux/amd64 -f Dockerfile.model-server .

push-model-server: check-uncommited
docker tag $(MODEL_SERVER_IMAGE_NAME) $(MODEL_REGISTERY)/$(MODEL_SERVER_IMAGE_NAME)
docker push $(MODEL_REGISTERY)/$(MODEL_SERVER_IMAGE_NAME)
docker tag $(MODEL_SERVER_IMAGE_NAME):$(TAG) $(MODEL_REGISTERY)/$(MODEL_SERVER_IMAGE_NAME):$(TAG)
docker push $(MODEL_REGISTERY)/$(MODEL_SERVER_IMAGE_NAME):$(TAG)

push-model-server-latest: check-uncommited
docker tag $(MODEL_SERVER_IMAGE_NAME):$(TAG) $(MODEL_REGISTERY)/$(MODEL_SERVER_IMAGE_NAME):latest
docker push $(MODEL_REGISTERY)/$(MODEL_SERVER_IMAGE_NAME):latest

# Used by GHA
build-push-model-server: build-model-server push-model-server
build-push-model-server: build-model-server push-model-server push-model-server-latest

build-pipeline-worker: check-uncommited
docker build -t $(PIPELINE_WORKER_IMAGE_NAME) --platform linux/amd64 -f Dockerfile.pipeline-worker .
docker build -t $(PIPELINE_WORKER_IMAGE_NAME):$(TAG) --platform linux/amd64 -f Dockerfile.pipeline-worker .

push-pipeline-worker: check-uncommited
docker tag $(PIPELINE_WORKER_IMAGE_NAME) $(MODEL_REGISTERY)/$(PIPELINE_WORKER_IMAGE_NAME)
docker push $(MODEL_REGISTERY)/$(PIPELINE_WORKER_IMAGE_NAME)
docker tag $(PIPELINE_WORKER_IMAGE_NAME):$(TAG) $(MODEL_REGISTERY)/$(PIPELINE_WORKER_IMAGE_NAME):$(TAG)
docker push $(MODEL_REGISTERY)/$(PIPELINE_WORKER_IMAGE_NAME):$(TAG)

push-pipeline-worker-latest: check-uncommited
docker tag $(PIPELINE_WORKER_IMAGE_NAME):$(TAG) $(MODEL_REGISTERY)/$(PIPELINE_WORKER_IMAGE_NAME):latest
docker push $(MODEL_REGISTERY)/$(PIPELINE_WORKER_IMAGE_NAME):latest

build-push-pipeline-worker: build-pipeline-worker push-pipeline-worker
build-push-pipeline-worker: build-pipeline-worker push-pipeline-worker push-pipeline-worker-latest

test-server:
python3 examples/test_server.py
@@ -86,7 +95,7 @@ run-dataflow:
--num_workers=8 \
--max_num_workers=8 \
--autoscaling_algorithm=THROUGHPUT_BASED \
--worker_harness_container_image=$(MODEL_REGISTERY)/$(PIPELINE_WORKER_IMAGE_NAME) \
--worker_harness_container_image=$(MODEL_REGISTERY)/$(PIPELINE_WORKER_IMAGE_NAME):$(TAG) \
--start "2024-07-11" \
--end "2024-07-11" \
--offset 0 \

0 comments on commit 8cd65f6

Please sign in to comment.