Skip to content

Commit

Permalink
add publish make target
Browse files Browse the repository at this point in the history
  • Loading branch information
pmhalvor committed Oct 20, 2024
1 parent 8cd65f6 commit 06ca412
Showing 1 changed file with 24 additions and 14 deletions.
38 changes: 24 additions & 14 deletions makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ TAG := $(VERSION)-$(GIT_SHA)
PIPELINE_IMAGE_NAME := whale-speech/pipeline
MODEL_SERVER_IMAGE_NAME := whale-speech/model-server
PIPELINE_WORKER_IMAGE_NAME := whale-speech/pipeline-worker
ENV_LOCATION := .env

PUBLIC_MODEL_SERVER_IMAGE_NAME := $(shell echo $(MODEL_SERVER_IMAGE_NAME) | sed 's/\//-/g')
PUBLIC_PIPELINE_WORKER_IMAGE_NAME := $(shell echo $(PIPELINE_WORKER_IMAGE_NAME) | sed 's/\//-/g')

local-run:
bash scripts/kill_model_server.sh
Expand All @@ -27,20 +29,15 @@ gcp-deduplicate:
python3 src/gcp.py --deduplicate

setup:
sudo apt-get update
sudo apt-get install python3-venv libhdf5-dev libsndfile1 gcc
python3 -m venv $(ENV_LOCATION)
$(ENV_LOCATION)/bin/pip install -r requirements/requirements.txt

run:
$(ENV_LOCATION)/bin/python3 src/pipeline.py

# Used by GHA
install:
conda create -n whale-speech python=3.11
conda activate whale-speech
sudo apt-get update
sudo apt-get install libhdf5-dev libsndfile1 gcc
python3 -m pip install -r requirements/requirements.txt
python3 -m pip install -r requirements/model-requirements.txt


# Docker related
check-uncommited:
git diff-index --quiet HEAD

Expand All @@ -53,6 +50,8 @@ push: check-uncommited

build-push: build push


# Model server related
build-model-server: check-uncommited
docker build -t $(MODEL_SERVER_IMAGE_NAME):$(TAG) --platform linux/amd64 -f Dockerfile.model-server .

Expand All @@ -64,9 +63,17 @@ push-model-server-latest: check-uncommited
docker tag $(MODEL_SERVER_IMAGE_NAME):$(TAG) $(MODEL_REGISTERY)/$(MODEL_SERVER_IMAGE_NAME):latest
docker push $(MODEL_REGISTERY)/$(MODEL_SERVER_IMAGE_NAME):latest

# Used by GHA
build-push-model-server: build-model-server push-model-server push-model-server-latest

publish-latest-model-server: build-push-model-server
docker tag $(MODEL_REGISTERY)/$(MODEL_SERVER_IMAGE_NAME):latest $(PUBLIC_MODEL_REGISTERY)/$(PUBLIC_MODEL_SERVER_IMAGE_NAME):latest
docker push $(PUBLIC_MODEL_REGISTERY)/$(PUBLIC_MODEL_SERVER_IMAGE_NAME):latest

test-server:
python3 examples/test_server.py


# Pipeline worker related
build-pipeline-worker: check-uncommited
docker build -t $(PIPELINE_WORKER_IMAGE_NAME):$(TAG) --platform linux/amd64 -f Dockerfile.pipeline-worker .

Expand All @@ -80,9 +87,12 @@ push-pipeline-worker-latest: check-uncommited

build-push-pipeline-worker: build-pipeline-worker push-pipeline-worker push-pipeline-worker-latest

test-server:
python3 examples/test_server.py
publish-latest-pipeline-worker: build-push-pipeline-worker
docker tag $(MODEL_REGISTERY)/$(PIPELINE_WORKER_IMAGE_NAME):latest $(PUBLIC_MODEL_REGISTERY)/$(PUBLIC_PIPELINE_WORKER_IMAGE_NAME):latest
docker push $(PUBLIC_MODEL_REGISTERY)/$(PUBLIC_PIPELINE_WORKER_IMAGE_NAME):latest


# Pipeline run related
run-dataflow:
python3 src/pipeline.py \
--job_name "whale-speech-$(GIT_SHA)" \
Expand Down

0 comments on commit 06ca412

Please sign in to comment.