diff --git a/.github/actions/install-deps-and-canopy/action.yml b/.github/actions/install-deps-and-canopy/action.yml index b1153a97..c1fc8785 100644 --- a/.github/actions/install-deps-and-canopy/action.yml +++ b/.github/actions/install-deps-and-canopy/action.yml @@ -9,11 +9,6 @@ inputs: description: "Whether to install canopy library, or dependencies only" required: true default: "true" - extras: - description: "Extra dependencies to install, space separated" - required: false - default: "cohere transformers" - runs: using: "composite" steps: @@ -41,12 +36,8 @@ runs: - name: Install dependencies shell: bash if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - env: - CANOPY_EXTRAS: ${{ inputs.extras }} - run: poetry install --no-interaction --no-root --with dev --extras "${CANOPY_EXTRAS}" + run: make install-extras POETRY_INSTALL_ARGS="--no-interaction --no-root --with dev" - name: Install project if: ${{ inputs.install-canopy == 'true' }} shell: bash - env: - CANOPY_EXTRAS: ${{ inputs.extras }} - run: make install-extras + run: make install-extras POETRY_INSTALL_ARGS="--with dev --no-interaction" diff --git a/.github/workflows/build-push-image.yml b/.github/workflows/build-push-image.yml index 45326c47..04975839 100644 --- a/.github/workflows/build-push-image.yml +++ b/.github/workflows/build-push-image.yml @@ -48,12 +48,18 @@ jobs: type=semver,pattern={{version}},enable=${{ github.event_name == 'push' }} type=raw,value=latest,enable=${{ github.event_name != 'push' }} type=raw,value=${{inputs.version}},enable=${{ github.event_name != 'push' }} + - name: Create build args + run: | + echo "POETRY_INSTALL_ARGS=$(make print-var VAR=POETRY_DEFAULT_EXTRAS)" >> $GITHUB_OUTPUT + id: build-args - name: Build and push uses: docker/build-push-action@v5 with: context: . platforms: linux/amd64 push: true + build-args: | + POETRY_INSTALL_ARGS=${{steps.build-args.outputs.POETRY_INSTALL_ARGS}} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} provenance: false diff --git a/Dockerfile b/Dockerfile index 13813cb0..471ad0e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,7 @@ ARG PYTHON_VERSION=3.11.7 ARG PORT=8000 +ARG POETRY_INSTALL_ARGS="" ################################ # PYTHON-BASE # Sets up all our shared environment variables @@ -63,9 +64,10 @@ WORKDIR /app COPY pyproject.toml ./ RUN poetry lock +ARG POETRY_INSTALL_ARGS # install runtime deps to VIRTUAL_ENV RUN --mount=type=cache,target=/root/.cache \ - poetry install --no-root --all-extras --only main + poetry install --no-root --only main $POETRY_INSTALL_ARGS ################################ @@ -78,13 +80,13 @@ WORKDIR /app COPY --from=builder-base /app/pyproject.toml pyproject.toml COPY --from=builder-base /app/poetry.lock poetry.lock - +ARG POETRY_INSTALL_ARGS # quicker install as runtime deps are already installed RUN --mount=type=cache,target=/root/.cache \ - poetry install --no-root --all-extras --with dev + poetry install --no-root --with dev $POETRY_INSTALL_ARGS COPY . . -RUN poetry install --all-extras --only-root +RUN poetry install --only-root $POETRY_INSTALL_ARGS ARG PORT EXPOSE $PORT @@ -101,7 +103,7 @@ FROM python-base as production ENV WORKER_COUNT=1 LABEL org.opencontainers.image.source="https://github.com/pinecone-io/canopy" -LABEL org.opencontainers.image.description="Image containing the canopy server." +LABEL org.opencontainers.image.description="Retrieval Augmented Generation (RAG) framework and context engine powered by Pinecone" LABEL org.opencontainers.image.licenses="Apache-2.0" RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ @@ -121,7 +123,8 @@ COPY --from=builder-base /app/poetry.lock poetry.lock COPY src/ src/ COPY config/ config/ RUN touch README.md -RUN poetry install --all-extras --only-root +ARG POETRY_INSTALL_ARGS +RUN poetry install --only-root $POETRY_INSTALL_ARGS ARG PORT EXPOSE $PORT diff --git a/Makefile b/Makefile index 0ad5b512..81e74c15 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,24 @@ TEST_WORKER_COUNT = 8 -REPOSITORY = ghcr.io/pinecone-io/canopy +POETRY_DEFAULT_EXTRAS = -E cohere -E transformers +POETRY_INSTALL_ARGS = +REPOSITORY = ghcr.io/pinecone-io/canopy IMAGE_TAG = $(shell poetry version -s) + CONTAINER_PORT = 8000 CONTAINER_ENV_FILE = .env CONTAINER_BUILD_DIR = . CONTAINER_BUILD_PLATFORM = linux/amd64 -CONTAINER_COMMON_BUILD_ARGS = --progress plain --platform $(CONTAINER_BUILD_PLATFORM) --build-arg PORT=$(CONTAINER_PORT) -CONTAINER_EXTRA_BUILD_ARGS = +CONTAINER_SYSTEM_BUILD_ARGS = --progress plain --platform $(CONTAINER_BUILD_PLATFORM) --build-arg PORT=$(CONTAINER_PORT) --build-arg POETRY_INSTALL_ARGS="$(POETRY_DEFAULT_EXTRAS) $(POETRY_INSTALL_ARGS)" +CONTAINER_BUILD_ARGS = # Only add the env file if it exists -CONTAINER_COMMON_RUN_ARGS = --platform linux/amd64 -p $(CONTAINER_PORT):$(CONTAINER_PORT) $(shell [ -e "$(CONTAINER_ENV_FILE)" ] && echo "--env-file $(CONTAINER_ENV_FILE)") -CONTAINER_EXTRA_RUN_ARGS = +CONTAINER_SYSTEM_RUN_ARGS = --platform linux/amd64 -p $(CONTAINER_PORT):$(CONTAINER_PORT) $(shell [ -e "$(CONTAINER_ENV_FILE)" ] && echo "--env-file $(CONTAINER_ENV_FILE)") +CONTAINER_RUN_ARGS = + -.PHONY: lint static install install-extras test test-unit test-system test-e2e install docker-build docker-build-dev docker-run docker-run-dev help +.PHONY: lint static install install-extras install-all-extras test test-unit test-system test-e2e docker-build docker-build-dev docker-run docker-run-dev print-var help lint: poetry run flake8 . @@ -23,10 +27,13 @@ static: poetry run mypy src install: - poetry install + poetry install $(POETRY_INSTALL_ARGS) install-extras: - poetry install --with dev --extras "$(CANOPY_EXTRAS)" + poetry install $(POETRY_DEFAULT_EXTRAS) $(POETRY_INSTALL_ARGS) + +install-all-extras: + poetry install --all-extras $(POETRY_INSTALL_ARGS) test: poetry run pytest -n $(TEST_WORKER_COUNT) --dist loadscope @@ -42,34 +49,43 @@ test-e2e: docker-build: @echo "Building Docker image..." - docker build $(CONTAINER_COMMON_BUILD_ARGS) $(CONTAINER_EXTRA_BUILD_ARGS) -t $(REPOSITORY):$(IMAGE_TAG) $(CONTAINER_BUILD_DIR) + docker build $(CONTAINER_SYSTEM_BUILD_ARGS) $(CONTAINER_BUILD_ARGS) -t $(REPOSITORY):$(IMAGE_TAG) $(CONTAINER_BUILD_DIR) @echo "Docker build complete." docker-build-dev: @echo "Building Docker image for development..." - docker build $(CONTAINER_COMMON_BUILD_ARGS) $(CONTAINER_EXTRA_BUILD_ARGS) -t $(REPOSITORY)-dev:$(IMAGE_TAG) --target=development $(CONTAINER_BUILD_DIR) + docker build $(CONTAINER_SYSTEM_BUILD_ARGS) $(CONTAINER_BUILD_ARGS) -t $(REPOSITORY)-dev:$(IMAGE_TAG) --target=development $(CONTAINER_BUILD_DIR) @echo "Development Docker build complete." docker-run: - docker run $(CONTAINER_COMMON_RUN_ARGS) $(CONTAINER_EXTRA_RUN_ARGS) $(REPOSITORY):$(IMAGE_TAG) + docker run $(CONTAINER_SYSTEM_RUN_ARGS) $(CONTAINER_RUN_ARGS) $(REPOSITORY):$(IMAGE_TAG) docker-run-dev: - docker run $(CONTAINER_COMMON_RUN_ARGS) $(CONTAINER_EXTRA_RUN_ARGS) -it $(REPOSITORY)-dev:$(IMAGE_TAG) + docker run $(CONTAINER_SYSTEM_RUN_ARGS) $(CONTAINER_RUN_ARGS) -it $(REPOSITORY)-dev:$(IMAGE_TAG) + +print-var: + @echo $($(VAR)) help: @echo "Available targets:" @echo "" @echo " -- DEV -- " - @echo " make install - Install all the dependencies." - @echo " make lint - Lint the code." - @echo " make static - Run static type checks." - @echo " make test - Test the code." - @echo " make test-unit - Run unit tests." - @echo " make test-system - Run system tests." - @echo " make test-e2e - Run e2e tests." + @echo " make install - Install only the required dependencies without any extras." + @echo " make install-extras - Install the dependencies with the default extras." + @echo " make install-all-extras - Install the dependencies with all extras." + @echo " make lint - Lint the code." + @echo " make static - Run static type checks." + @echo " make test - Test the code." + @echo " make test-unit - Run unit tests." + @echo " make test-system - Run system tests." + @echo " make test-e2e - Run e2e tests." @echo "" @echo " -- DOCKER -- " - @echo " make docker-build - Build the Docker image." - @echo " make docker-build-dev - Build the Docker image for development." - @echo " make docker-run - Run the Docker image." - @echo " make docker-run-dev - Run the Docker image for development." + @echo " make docker-build - Build the Docker image." + @echo " make docker-build-dev - Build the Docker image for development." + @echo " make docker-run - Run the Docker image." + @echo " make docker-run-dev - Run the Docker image for development." + @echo "" + @echo " -- MISC -- " + @echo " make print-var VAR= - Print the value of a variable." +