Add gpu image

.github/workflows/backend-base-gpu.yml

@@ -0,0 +1,51 @@
+name: backend-base
+
+on:
+  push:
+    # Publish `dev` as Docker `latest` image.
+    branches:
+      - main
+    paths:
+      - "backend-gpu/base/**"
+
+jobs:
+  # Run tests.
+  # See also https://docs.docker.com/docker-hub/builds/automated-testing/
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Run tests
+        run: echo "To-Do Add Running Tests"
+
+  build:
+    # Ensure test job passes before pushing image.
+    needs: test
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: ./backend-gpu/base
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: reallibrephotos/librephotos-base-gpu:dev

.github/workflows/backend-gpu.yml

@@ -0,0 +1,94 @@
+name: backend
+
+on:
+  release:
+    types: [created]
+  push:
+    branches:
+      - main
+    paths:
+      - "backend-gpu/**"
+  repository_dispatch:
+    types: backend-commit-event
+
+env:
+  IMAGE_NAME: librephotos
+
+jobs:
+  # Run tests.
+  # See also https://docs.docker.com/docker-hub/builds/automated-testing/
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Run tests
+        run: echo "To-Do Add Running Tests"
+
+  build:
+    name: Building dev build
+    # Ensure test job passes before pushing image.
+    if: github.event_name != 'release'
+    needs: test
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: ./backend-gpu
+          build-args: |
+            "IMAGE_TAG=dev"
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: reallibrephotos/librephotos-gpu:dev
+
+  release-job:
+    name: Releasing
+    if: github.event_name == 'release' && github.event.action == 'created'
+    # Ensure test job passes before pushing image.
+    needs: test
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: ./backend-gpu
+          build-args: |
+            "IMAGE_TAG=${{ github.event.release.tag_name}}"
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: reallibrephotos/librephotos-gpu:${{ github.event.release.tag_name }},reallibrephotos/librephotos-gpu:latest

backend-gpu/Dockerfile

@@ -0,0 +1,16 @@
+FROM reallibrephotos/librephotos-base-gpu:dev
+# actual project
+ARG DEBUG
+ARG IMAGE_TAG
+ENV IMAGE_TAG ${IMAGE_TAG}
+
+WORKDIR /code
+RUN git clone --depth 1 https://github.com/LibrePhotos/librephotos .
+RUN pip install --break-system-packages --no-cache-dir -r requirements.txt
+RUN if [ "$DEBUG" = 1 ] ; then \
+  pip install --break-system-packages -r requirements.dev.txt; \
+  fi
+EXPOSE 8001
+
+COPY entrypoint.sh /entrypoint.sh
+CMD ["/entrypoint.sh"]

backend-gpu/base/Dockerfile

@@ -0,0 +1,113 @@
+FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04
+ARG TARGETPLATFORM
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install python
+RUN apt-get update \
+	&& apt-get install -y python3-pip python3-dev \
+	&& cd /usr/local/bin \
+	&& ln -s /usr/bin/python3 python \
+	&& pip3 install --upgrade --break-system-packages pip \
+	&& apt-get clean \
+	&& rm -rf /var/lib/apt/lists/*
+
+# System packages installation
+RUN apt-get update && \
+	apt-get install -y --no-install-recommends \
+	build-essential \
+	bzip2 \
+	brotli \
+	curl \
+	ffmpeg \
+	git \
+	libboost-all-dev \
+	libcfitsio-dev \ 
+	libexif-dev \
+	libexpat-dev \
+	libexpat1-dev \ 
+	libgif-dev \
+	libgl1-mesa-glx \
+	libglib2.0-dev \
+	libgsf-1-dev \ 
+	libheif-dev \
+	libimage-exiftool-perl \
+	libimagequant-dev \
+	libjpeg-dev \
+	liblapack-dev \
+	liblcms2-dev \
+	libmagic1 \
+	libopenblas-dev \
+	libopenexr-dev \ 
+	liborc-dev \
+	libpng-dev \
+	libpq-dev \
+	librsvg2-dev \
+	libsm6 \
+	libtiff5-dev \ 
+	libtool \ 
+	libtool-bin \
+	libwebp-dev \
+	libxrender-dev \
+	pkg-config \ 
+	rustc \
+	cargo \
+	libtinfo5 \
+	libjxl-dev \
+	libvips-dev \
+	swig && \
+	apt-get clean && \
+	rm -rf /var/lib/apt/lists/*
+
+# Install ExifTool
+WORKDIR /tmp/exiftool
+RUN curl -SL https://exiftool.org/Image-ExifTool-12.64.tar.gz | tar -xz && \
+	cd Image-ExifTool-12.64 && \
+	perl Makefile.PL && \
+	make install && \
+	rm -rf /tmp/exiftool
+
+# Current Version: pytorch 2.0.1
+RUN pip3 install --no-cache-dir --break-system-packages torch torchvision
+
+RUN pip3 install --no-cache-dir --break-system-packages cmake==3.21.2
+
+#Build and install libraw, supports more RAW image types
+WORKDIR /tmp/builds
+RUN git clone https://github.com/LibRaw/LibRaw && \
+	cd LibRaw && \ 
+	autoreconf --install && \
+	./configure && \
+	make && \
+	make install && \
+	rm -rf /tmp/builds/*
+
+#Build and install imagemagick to use current libraw
+WORKDIR /tmp/builds
+ARG IMAGEMAGICK_VERSION=7.1.1-8
+RUN curl -SL https://imagemagick.org/archive/releases/ImageMagick-${IMAGEMAGICK_VERSION}.tar.xz | tar -xJf- && \
+	cd ImageMagick-${IMAGEMAGICK_VERSION} && \
+	./configure --with-modules && \
+	make install && \
+	ldconfig /usr/local/lib && \
+	rm -rf /tmp/builds/*
+
+# Build and install dlib
+# Compile it WITHOUT AVX and SSE4 instructions to ensure compatibility
+WORKDIR /tmp/builds
+RUN git clone --depth 1 --branch 'master' https://github.com/davisking/dlib.git && \
+	mkdir dlib/build && \
+	cd dlib/build && \
+	cmake .. -DDLIB_USE_CUDA=1 -DUSE_AVX_INSTRUCTIONS=1 -DLIB_NO_GUI_SUPPORT=0 && \
+	cmake --build . && \
+	cd /tmp/builds/dlib && \
+	python3 setup.py install --no USE_AVX_INSTRUCTIONS --yes DLIB_USE_CUDA --no USE_SSE4_INSTRUCTIONS && \
+	rm -rf /tmp/builds/* 
+
+# Download pretrained models
+WORKDIR /data_models
+RUN mkdir -p /root/.cache/torch/hub/checkpoints/ && \
+	curl -SL https://github.com/LibrePhotos/librephotos-docker/releases/download/0.1/places365.tar.gz | tar -zxC /data_models/ && \
+	curl -SL https://github.com/LibrePhotos/librephotos-docker/releases/download/0.1/im2txt.tar.gz | tar -zxC /data_models/ && \
+	curl -SL https://github.com/LibrePhotos/librephotos-docker/releases/download/0.1/clip-embeddings.tar.gz | tar -zxC /data_models/ && \
+	curl -SL https://download.pytorch.org/models/resnet152-b121ed2d.pth -o /root/.cache/torch/hub/checkpoints/resnet152-b121ed2d.pth
+

backend-gpu/entrypoint.sh

@@ -0,0 +1,39 @@
+#! /bin/bash
+export PYTHONUNBUFFERED=TRUE
+export PYTHONFAULTHANDLER=1
+
+if [[ "$(uname -m)" == "aarch64"* ]]; then
+  export OPENBLAS_CORETYPE=ARMV8
+  echo "ARM architecture detected. OPENBLAS_CORETYPE set to ARMV8"
+fi
+export OPENBLAS_NUM_THREADS=1 
+export OPENBLAS_MAIN_FREE=1
+
+mkdir -p /logs
+
+python manage.py showmigrations | tee /logs/show_migrate.log
+python manage.py migrate | tee /logs/command_migrate.log
+python manage.py showmigrations | tee /logs/show_migrate.log
+python manage.py collectstatic --no-input
+python image_similarity/main.py 2>&1 | tee /logs/gunicorn_image_similarity.log &
+python service/thumbnail/main.py 2>&1 | tee /logs/gunicorn_thumbnail.log &
+python manage.py clear_cache 
+python manage.py build_similarity_index 2>&1 | tee /logs/command_build_similarity_index.log
+
+if [ -n "$ADMIN_USERNAME" ]
+then
+    python manage.py createadmin -u $ADMIN_USERNAME $ADMIN_EMAIL 2>&1 | tee /logs/command_createadmin.log
+fi
+
+echo "Running backend server..."
+
+python manage.py qcluster 2>&1 | tee /logs/qcluster.log &
+
+if [ "$DEBUG" = 1 ]
+then
+    echo "development backend starting"
+    gunicorn --worker-class=gevent --reload --bind 0.0.0.0:8001 --log-level=info ownphotos.wsgi 2>&1 | tee /logs/gunicorn_django.log
+else
+    echo "production backend starting"
+    gunicorn --worker-class=gevent --bind 0.0.0.0:8001 --log-level=info ownphotos.wsgi 2>&1 | tee /logs/gunicorn_django.log
+fi