Change the default poetry extras (#277)

* Change the default poetry extras * Fix comments * Fix
pinecone-io · Feb 8, 2024 · 84351dc · 84351dc
1 parent a2cf274
commit 84351dc
Show file tree

Hide file tree

Showing 4 changed files with 56 additions and 40 deletions.
diff --git a/.github/actions/install-deps-and-canopy/action.yml b/.github/actions/install-deps-and-canopy/action.yml
@@ -9,11 +9,6 @@ inputs:
     description: "Whether to install canopy library, or dependencies only"
     required: true
     default: "true"
-  extras:
-    description: "Extra dependencies to install, space separated"
-    required: false
-    default: "cohere transformers"
-
 runs:
   using: "composite"
   steps:
@@ -41,12 +36,8 @@ runs:
   - name: Install dependencies
     shell: bash
     if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-    env:
-      CANOPY_EXTRAS: ${{ inputs.extras }}
-    run: poetry install --no-interaction --no-root --with dev --extras "${CANOPY_EXTRAS}"
+    run: make install-extras POETRY_INSTALL_ARGS="--no-interaction --no-root --with dev"
   - name: Install project
     if: ${{ inputs.install-canopy == 'true' }}
     shell: bash
-    env:
-      CANOPY_EXTRAS: ${{ inputs.extras }}
-    run: make install-extras
+    run: make install-extras POETRY_INSTALL_ARGS="--with dev --no-interaction"
diff --git a/.github/workflows/build-push-image.yml b/.github/workflows/build-push-image.yml
@@ -48,12 +48,18 @@ jobs:
             type=semver,pattern={{version}},enable=${{ github.event_name == 'push' }}
             type=raw,value=latest,enable=${{ github.event_name != 'push' }}
             type=raw,value=${{inputs.version}},enable=${{ github.event_name != 'push' }}
+      - name: Create build args
+        run: |
+          echo "POETRY_INSTALL_ARGS=$(make print-var VAR=POETRY_DEFAULT_EXTRAS)" >> $GITHUB_OUTPUT
+        id: build-args
       - name: Build and push
         uses: docker/build-push-action@v5
         with:
           context: .
           platforms: linux/amd64
           push: true
+          build-args: |
+            POETRY_INSTALL_ARGS=${{steps.build-args.outputs.POETRY_INSTALL_ARGS}}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           provenance: false

diff --git a/Dockerfile b/Dockerfile
@@ -3,6 +3,7 @@
 
 ARG PYTHON_VERSION=3.11.7
 ARG PORT=8000
+ARG POETRY_INSTALL_ARGS=""
 ################################
 # PYTHON-BASE
 # Sets up all our shared environment variables
@@ -63,9 +64,10 @@ WORKDIR /app
 COPY pyproject.toml ./
 RUN poetry lock
 
+ARG POETRY_INSTALL_ARGS
 # install runtime deps to VIRTUAL_ENV
 RUN --mount=type=cache,target=/root/.cache \
-    poetry install --no-root --all-extras --only main
+    poetry install --no-root --only main $POETRY_INSTALL_ARGS
 
 
 ################################
@@ -78,13 +80,13 @@ WORKDIR /app
 COPY --from=builder-base /app/pyproject.toml pyproject.toml
 COPY --from=builder-base /app/poetry.lock poetry.lock
 
-
+ARG POETRY_INSTALL_ARGS
 # quicker install as runtime deps are already installed
 RUN --mount=type=cache,target=/root/.cache \
-    poetry install --no-root --all-extras --with dev
+    poetry install --no-root --with dev $POETRY_INSTALL_ARGS
 
 COPY . .
-RUN poetry install --all-extras --only-root
+RUN poetry install --only-root $POETRY_INSTALL_ARGS
 
 ARG PORT
 EXPOSE $PORT
@@ -101,7 +103,7 @@ FROM python-base as production
 ENV WORKER_COUNT=1
 
 LABEL org.opencontainers.image.source="https://github.com/pinecone-io/canopy"
-LABEL org.opencontainers.image.description="Image containing the canopy server."
+LABEL org.opencontainers.image.description="Retrieval Augmented Generation (RAG) framework and context engine powered by Pinecone"
 LABEL org.opencontainers.image.licenses="Apache-2.0"
 
 RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
@@ -121,7 +123,8 @@ COPY --from=builder-base /app/poetry.lock poetry.lock
 COPY src/ src/
 COPY config/ config/
 RUN touch README.md
-RUN poetry install --all-extras --only-root
+ARG POETRY_INSTALL_ARGS
+RUN poetry install --only-root $POETRY_INSTALL_ARGS
 
 ARG PORT
 EXPOSE $PORT

diff --git a/Makefile b/Makefile
@@ -1,20 +1,24 @@
 TEST_WORKER_COUNT = 8
 
-REPOSITORY = ghcr.io/pinecone-io/canopy
+POETRY_DEFAULT_EXTRAS = -E cohere -E transformers
+POETRY_INSTALL_ARGS =
 
+REPOSITORY = ghcr.io/pinecone-io/canopy
 IMAGE_TAG = $(shell poetry version -s)
+
 CONTAINER_PORT = 8000
 CONTAINER_ENV_FILE = .env
 CONTAINER_BUILD_DIR = .
 CONTAINER_BUILD_PLATFORM = linux/amd64
-CONTAINER_COMMON_BUILD_ARGS = --progress plain --platform $(CONTAINER_BUILD_PLATFORM) --build-arg PORT=$(CONTAINER_PORT)
-CONTAINER_EXTRA_BUILD_ARGS =
+CONTAINER_SYSTEM_BUILD_ARGS = --progress plain --platform $(CONTAINER_BUILD_PLATFORM) --build-arg PORT=$(CONTAINER_PORT) --build-arg POETRY_INSTALL_ARGS="$(POETRY_DEFAULT_EXTRAS) $(POETRY_INSTALL_ARGS)"
+CONTAINER_BUILD_ARGS =
 
 # Only add the env file if it exists
-CONTAINER_COMMON_RUN_ARGS = --platform linux/amd64 -p $(CONTAINER_PORT):$(CONTAINER_PORT) $(shell [ -e "$(CONTAINER_ENV_FILE)" ] && echo "--env-file $(CONTAINER_ENV_FILE)")
-CONTAINER_EXTRA_RUN_ARGS =
+CONTAINER_SYSTEM_RUN_ARGS = --platform linux/amd64 -p $(CONTAINER_PORT):$(CONTAINER_PORT) $(shell [ -e "$(CONTAINER_ENV_FILE)" ] && echo "--env-file $(CONTAINER_ENV_FILE)")
+CONTAINER_RUN_ARGS =
+
 
-.PHONY: lint static install install-extras test test-unit test-system test-e2e install docker-build docker-build-dev docker-run docker-run-dev help
+.PHONY: lint static install install-extras install-all-extras test test-unit test-system test-e2e docker-build docker-build-dev docker-run docker-run-dev print-var help
 
 lint:
 	poetry run flake8 .
@@ -23,10 +27,13 @@ static:
 	poetry run mypy src
 
 install:
-	poetry install
+	poetry install $(POETRY_INSTALL_ARGS)
 
 install-extras:
-	poetry install --with dev --extras "$(CANOPY_EXTRAS)"
+	poetry install $(POETRY_DEFAULT_EXTRAS) $(POETRY_INSTALL_ARGS)
+
+install-all-extras:
+	poetry install --all-extras $(POETRY_INSTALL_ARGS)
 
 test:
 	poetry run pytest -n $(TEST_WORKER_COUNT) --dist loadscope
@@ -42,34 +49,43 @@ test-e2e:
 
 docker-build:
 	@echo "Building Docker image..."
-	docker build $(CONTAINER_COMMON_BUILD_ARGS) $(CONTAINER_EXTRA_BUILD_ARGS) -t $(REPOSITORY):$(IMAGE_TAG) $(CONTAINER_BUILD_DIR)
+	docker build $(CONTAINER_SYSTEM_BUILD_ARGS) $(CONTAINER_BUILD_ARGS) -t $(REPOSITORY):$(IMAGE_TAG) $(CONTAINER_BUILD_DIR)
 	@echo "Docker build complete."
 
 docker-build-dev:
 	@echo "Building Docker image for development..."
-	docker build $(CONTAINER_COMMON_BUILD_ARGS) $(CONTAINER_EXTRA_BUILD_ARGS) -t $(REPOSITORY)-dev:$(IMAGE_TAG) --target=development $(CONTAINER_BUILD_DIR)
+	docker build $(CONTAINER_SYSTEM_BUILD_ARGS) $(CONTAINER_BUILD_ARGS) -t $(REPOSITORY)-dev:$(IMAGE_TAG) --target=development $(CONTAINER_BUILD_DIR)
 	@echo "Development Docker build complete."
 
 docker-run:
-	docker run $(CONTAINER_COMMON_RUN_ARGS) $(CONTAINER_EXTRA_RUN_ARGS) $(REPOSITORY):$(IMAGE_TAG)
+	docker run $(CONTAINER_SYSTEM_RUN_ARGS) $(CONTAINER_RUN_ARGS) $(REPOSITORY):$(IMAGE_TAG)
 
 docker-run-dev:
-	docker run $(CONTAINER_COMMON_RUN_ARGS) $(CONTAINER_EXTRA_RUN_ARGS) -it $(REPOSITORY)-dev:$(IMAGE_TAG)
+	docker run $(CONTAINER_SYSTEM_RUN_ARGS) $(CONTAINER_RUN_ARGS) -it $(REPOSITORY)-dev:$(IMAGE_TAG)
+
+print-var:
+	@echo $($(VAR))
 
 help:
 	@echo "Available targets:"
 	@echo ""
 	@echo " -- DEV -- "
-	@echo "  make install            - Install all the dependencies."
-	@echo "  make lint               - Lint the code."
-	@echo "  make static             - Run static type checks."
-	@echo "  make test               - Test the code."
-	@echo "  make test-unit          - Run unit tests."
-	@echo "  make test-system        - Run system tests."
-	@echo "  make test-e2e           - Run e2e tests."
+	@echo "  make install                     - Install only the required dependencies without any extras."
+	@echo "  make install-extras              - Install the dependencies with the default extras."
+	@echo "  make install-all-extras          - Install the dependencies with all extras."
+	@echo "  make lint                        - Lint the code."
+	@echo "  make static                      - Run static type checks."
+	@echo "  make test                        - Test the code."
+	@echo "  make test-unit                   - Run unit tests."
+	@echo "  make test-system                 - Run system tests."
+	@echo "  make test-e2e                    - Run e2e tests."
 	@echo ""
 	@echo " -- DOCKER -- "
-	@echo "  make docker-build       - Build the Docker image."
-	@echo "  make docker-build-dev   - Build the Docker image for development."
-	@echo "  make docker-run         - Run the Docker image."
-	@echo "  make docker-run-dev     - Run the Docker image for development."
+	@echo "  make docker-build                - Build the Docker image."
+	@echo "  make docker-build-dev            - Build the Docker image for development."
+	@echo "  make docker-run                  - Run the Docker image."
+	@echo "  make docker-run-dev              - Run the Docker image for development."
+	@echo ""
+	@echo " -- MISC -- "
+	@echo "  make print-var VAR=<variable>    - Print the value of a variable."
+