Add Dockerfile and build image flow (#567)

* Add Dockerfile * Make ibis version same as java engine * Add CI Install python dependencies in the CI * Make python dependencies install in the Docker If we install the dependencies in the CI, may cause platform issue * Use docker/build-push-action * Fix tag name * Add docker/metadata-action * Fix Dockerfile path * Fix context * Fix tag * Update README * Remove invalid default working-directory * Fix pyproject.toml path * Use step.working-directory * Use poetry version * Add empty line in the end of file * Edit title
Canner · May 28, 2024 · c278311 · c278311
1 parent c251242
commit c278311
Show file tree

Hide file tree

Showing 7 changed files with 235 additions and 16 deletions.
diff --git a/.github/workflows/build-dev-image.yml b/.github/workflows/build-dev-image.yml
@@ -47,3 +47,32 @@ jobs:
             --tag ghcr.io/canner/wren-engine:nightly \
             --push -f ./Dockerfile \
             --build-arg "WREN_VERSION=${WREN_VERSION}" .
+  build-ibis-image:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/canner/wren-engine-ibis
+          tags: |
+            type=sha
+            type=raw,value=nightly
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: ./ibis-server
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml
@@ -8,7 +8,23 @@ on:
         description: Docker image tag name (Optional)
 
 jobs:
+  prepare-tag:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Prepare tag name
+        id: prepare_tag
+        run: |
+          if [ -n "${{ github.event.inputs.docker_image_tag_name }}" ]; then
+            tag_name=${{ github.event.inputs.docker_image_tag_name }}
+          else
+            tag_name=$(echo ${{ github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')-$(git log -1 --pretty=%h)
+          fi
+          echo "tag_name=$tag_name" >> "$GITHUB_OUTPUT"
+    outputs:
+      tag_name: ${{ steps.prepare_tag.outputs.tag_name }}
   build-image:
+    needs: prepare-tag
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -31,23 +47,45 @@ jobs:
         uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
-      - name: Prepare tag name
-        id: prepare_tag
-        run: |
-          if [ -n "${{ github.event.inputs.docker_image_tag_name }}" ]; then
-            tag_name=${{ github.event.inputs.docker_image_tag_name }}
-          else
-            tag_name=$(echo ${{ github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')-$(git log -1 --pretty=%h)
-          fi
-          echo ::set-output name=tag_name::$tag_name
       - name: Build Docker image
+        env:
+          TAG_NAME: ${{ needs.prepare-tag.outputs.tag_name }}
         run: |
           WREN_VERSION=$(./mvnw --quiet help:evaluate -Dexpression=project.version -DforceStdout)
           cd ./docker
           cp ../wren-server/target/wren-server-${WREN_VERSION}-executable.jar ./
           cp -r ../wren-sqlglot-server/ ./
           docker buildx build \
             --platform linux/amd64,linux/arm64 \
-            --tag ghcr.io/canner/wren-engine:${{ steps.prepare_tag.outputs.tag_name }} \
+            --tag ghcr.io/canner/wren-engine:$TAG_NAME \
             --push -f ./Dockerfile \
             --build-arg "WREN_VERSION=${WREN_VERSION}" .
+  build-ibis-image:
+    needs: prepare-tag
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/canner/wren-engine-ibis
+          tags: |
+            type=raw,value=${{ needs.prepare-tag.outputs.tag_name }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: ./ibis-server
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml
@@ -8,7 +8,7 @@ on:
         description: Specific version number (Optional). Default will be the current version plus 0.0.1.
 
 jobs:
-  prepare-release:
+  stable-release-wren-engine:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -62,3 +62,65 @@ jobs:
             --tag ghcr.io/canner/wren-engine:latest \
             --push -f ./Dockerfile \
             --build-arg "WREN_VERSION=${WREN_VERSION}" .
+  prepare-ibis-version:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GHCR_TOKEN }}
+      - name: Set up Git
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "stable-release-bot"
+      - uses: actions/setup-python@v5
+        with:
+          python-version-file: ./ibis-server/pyproject.toml
+      - uses: abatilo/actions-poetry@v3
+        with:
+          poetry-version: 1.7.1
+      - name: Prepare next version
+        id: next_version
+        working-directory: ibis-server
+        run: |
+          if [ -n "${{ github.event.inputs.specific_version }}" ]; then
+            poetry version --next-phase ${{ github.event.inputs.specific_version }}
+          else
+            poetry version patch
+          fi
+          git add pyproject.toml
+          git commit -m "Upgrade ibis version to $version"
+          git push
+          version=$(poetry version | awk '{print $2}')
+          echo "value=$version" >> $GITHUB_OUTPUT
+    outputs:
+      next_version: ${{ steps.next_version.outputs.value }}
+  stable-release-ibis:
+    needs: prepare-ibis-version
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/canner/wren-engine-ibis
+          tags: |
+            type=raw,value=${{ needs.prepare-ibis-version.outputs.next_version }}
+            type=raw,value=latest
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: ./ibis-server
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
diff --git a/ibis-server/Dockerfile b/ibis-server/Dockerfile
@@ -0,0 +1,41 @@
+FROM python:3.11-buster as builder
+
+# libpq-dev is required for psycopg2
+RUN apt-get update && apt-get -y install libpq-dev
+
+    # python
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    # pip
+    PIP_NO_CACHE_DIR=off \
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    PIP_DEFAULT_TIMEOUT=100 \
+    # poetry
+    POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1
+
+RUN pip install poetry==1.7.1
+
+WORKDIR /app
+
+COPY pyproject.toml ./
+
+RUN poetry install --without dev
+
+FROM python:3.11-slim-buster as runtime
+
+# libpq-dev is required for psycopg2
+RUN apt-get update  \
+    && apt-get -y install libpq-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV VIRTUAL_ENV=/app/.venv \
+    PATH="/app/.venv/bin:$PATH"
+
+COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+COPY app app
+
+EXPOSE 8000
+
+ENTRYPOINT fastapi run
diff --git a/ibis-server/Makefile b/ibis-server/Makefile
@@ -13,3 +13,9 @@ dev:
 
 test:
 	poetry run pytest
+
+docker-build:
+	docker image build . -t wren-engine-ibis
+
+docker-run:
+	docker run -it --rm -p 8000:8000 --env-file .env wren-engine-ibis
diff --git a/ibis-server/README.md b/ibis-server/README.md
@@ -1,16 +1,59 @@
 # Ibis server
 
-## Environment Setup
+## Quick Start
+### Running on Docker
+Pull the image from the GitHub Container Registry
+```bash
+docker pull ghcr.io/canner/wren-engine-ibis:latest
+```
+Create `.env` file and fill in the environment variables (see [Environment Variables](#Environment-Variables)) \
+```bash
+vim .env
+```
+Run the container
+```bash
+docker run --env-file .env -p 8000:8000 ghcr.io/canner/wren-engine-ibis:latest
+```
+### Running on Local
+Requirements:
+- Python 3.11
+- [poetry](https://github.com/python-poetry/poetry) 1.7.1
+
+Clone the repository and navigate to the ibis directory
+```bash
+git clone [email protected]:Canner/wren-engine.git
+cd ibis-server
+```
+Create `.env` file and fill in the environment variables (see [Environment Variables](#Environment-Variables))
+```bash
+vim .env
+```
+Install the dependencies
+```bash
+make install
+```
+Run the server
+```bash
+make run
+```
+
+## Developer Guide
+
+### Environment Setup
 - Python 3.11
 - Install `poetry` with version 1.7.1: `curl -sSL https://install.python-poetry.org | python3 - --version 1.7.1`
 - Execute `make install` to install the dependencies
 - Execute `make test` to run the tests
 - Create `.env` file and fill in the environment variables
 
-## Environment Variables
+### Environment Variables
 - `WREN_ENGINE_ENDPOINT`: The endpoint of the Wren engine
 
-## Start the server
+### Start the server
 - Execute `make run` to start the server
 - Execute `make dev` to start the server in development mode. It will auto-reload after the code is edited.
 - Default port is `8000`, you can change it by `make run PORT=8001` or `make dev PORT=8001`
+
+### Docker
+- Build the image: `make docker-build`
+- Run the container: `make docker-run`
diff --git a/ibis-server/pyproject.toml b/ibis-server/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ibis-server"
-version = "0.1.0"
+version = "0.4.5"
 description = ""
 authors = ["Canner <[email protected]>"]
 readme = "README.md"
@@ -15,7 +15,7 @@ ibis-framework = {extras = ["bigquery", "postgres", "snowflake"], version = "9.0
 google-auth = "2.29.0"
 httpx = "0.27.0"
 python-dotenv = "1.0.1"
-orjson = "2.0.1"
+orjson = "3.10.3"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "8.2.0"