Merge branch 'main' into pleary-dev

.flake8

@@ -1,7 +1,6 @@
 [flake8]
 ignore = D203
 max-line-length = 100
-inline-quotes = "
 exclude =
     .git,
     __pycache__,
@@ -12,3 +11,4 @@ exclude =
     test-obs*,
     venv
 max-complexity = 10
+inline-quotes = "

.github/workflows/CI-build-test.yml

@@ -0,0 +1,32 @@
+name: Vision Build and Test
+
+on:
+  workflow_call:
+
+jobs:
+  build-and-test:
+    name: Build/Test 
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version:
+          - 3.8.10
+          - 3.11.6
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Use Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install -r requirements-test.txt
+
+    - name: Test with pytest
+      run: |
+        coverage run -m pytest -s && coverage report --show-missing

.github/workflows/CI-pre-build.yml

@@ -0,0 +1,14 @@
+name: Pre Build
+
+on:
+  workflow_call:
+
+jobs:
+  pre-build:
+    name: Pre-build
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: fkirc/skip-duplicate-actions@master
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        cancel_others: true

.github/workflows/CICD-dev.yml

@@ -7,39 +7,53 @@ on:
       - '!main'
 
 jobs:
+  pre-build:
+    uses: ./.github/workflows/CI-pre-build.yml
+
   build-and-test:
-    name: Build/Test
-    runs-on: ubuntu-20.04
-    steps:
-    - uses: actions/checkout@v4
+    needs: pre-build
+    uses: ./.github/workflows/CI-build-test.yml
 
   build-and-push-dev-docker-image:
-    name: Build/Push Dev Docker Image
+    name: Build/Push Dev Docker Image 
+    needs: pre-build
     runs-on: ubuntu-20.04
-    steps:
+    steps:    
     - uses: actions/checkout@v4
 
     - name: Login to Docker Hub
       uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-
+        password: ${{ secrets.DOCKERHUB_TOKEN }}  
+    
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
-
-    - name: Build/Push dev vision-api
+    
+    - name: Build/Push Dev inatvisionapi
       uses: docker/build-push-action@v5
       with:
         context: .
         file: ./Dockerfile
         push: true
-        tags: ${{ secrets.DOCKERHUB_USERNAME }}/vision-api:${{ github.ref_name }}
+        tags: ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi:${{ github.ref_name }}
 
-    - name: Build/Push dev vision-cleanup
+    - name: Build/Push Dev inatvisionapi-cleanup
       uses: docker/build-push-action@v5
       with:
         context: .
         file: ./Dockerfile-cleanup
         push: true
-        tags: ${{ secrets.DOCKERHUB_USERNAME }}/vision-cleanup:${{ github.ref_name }}
+        tags: ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi-cleanup:${{ github.ref_name }}
+
+  notify-slack:
+    name: Notify Slack
+    needs: build-and-push-dev-docker-image
+    if: ${{ success() || failure() }}
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: iRoachie/[email protected]
+      if: env.SLACK_WEBHOOK_URL != null
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BUILDS_WEBHOOK_URL }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/CICD-main.yml

@@ -6,39 +6,57 @@ on:
       - main
 
 jobs:
+  pre-build:
+    uses: ./.github/workflows/CI-pre-build.yml
+
   build-and-test:
-    name: Build/Test
-    runs-on: ubuntu-20.04
-    steps:
-    - uses: actions/checkout@v4
+    needs: pre-build
+    uses: ./.github/workflows/CI-build-test.yml
 
-  build-and-push-staging-docker-image:
-    name: Build/Push Staging Docker Image
+  build-and-push-main-docker-image:
+    name: Build/Push Main Docker Image 
+    needs: pre-build
     runs-on: ubuntu-20.04
-    steps:
+    steps:    
     - uses: actions/checkout@v4
 
     - name: Login to Docker Hub
       uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-
+        password: ${{ secrets.DOCKERHUB_TOKEN }}  
+    
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
-
-    - name: Build/Push staging vision-api
+    
+    - name: Build/Push Main inatvisionapi
       uses: docker/build-push-action@v5
       with:
         context: .
         file: ./Dockerfile
         push: true
-        tags: ${{ secrets.DOCKERHUB_USERNAME }}/vision-api:${{ github.sha }}
+        tags: |
+          ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi:${{ github.sha }}
+          ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi:latest
 
-    - name: Build/Push staging vision-cleanup
+    - name: Build/Push Main inatvisionapi-cleanup
       uses: docker/build-push-action@v5
       with:
         context: .
         file: ./Dockerfile-cleanup
         push: true
-        tags: ${{ secrets.DOCKERHUB_USERNAME }}/vision-cleanup:${{ github.sha }}
+        tags: |
+          ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi-cleanup:${{ github.sha }}
+          ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi-cleanup:latest
+
+  notify-slack:
+    name: Notify Slack
+    needs: build-and-push-main-docker-image
+    if: ${{ success() || failure() }}
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: iRoachie/[email protected]
+      if: env.SLACK_WEBHOOK_URL != null
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BUILDS_WEBHOOK_URL }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/CICD-pr.yml

@@ -0,0 +1,20 @@
+name: Vision CI/CD Pull Request
+
+on:
+  pull_request:
+
+jobs:
+  build-and-test:
+    uses: ./.github/workflows/CI-build-test.yml
+
+  notify-slack:
+    name: Notify Slack
+    needs: build-and-test
+    if: ${{ success() || failure() }}
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: iRoachie/[email protected]
+      if: env.SLACK_WEBHOOK_URL != null
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BUILDS_WEBHOOK_URL }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Dockerfile

@@ -9,13 +9,13 @@ ENV PATH="/home/inaturalist/.local/bin:${PATH}"
 
 RUN pip install --upgrade pip
 
-# set the working directory in the container
+# Set the working directory in the container
 WORKDIR /home/inaturalist/vision
 
-# copy the dependencies file to the working directory
-COPY --chown=inaturalist:inaturalist ./requirements-production.txt /home/inaturalist/vision/requirements.txt
+# Copy the dependencies file to the working directory
+COPY --chown=inaturalist:inaturalist ./requirements.txt /home/inaturalist/vision/requirements.txt
 
-# install dependencies
+# Install dependencies
 RUN UWSGI_EMBED_PLUGINS=stats_pusher_statsd pip install -r requirements.txt
 
 # Copy app and libs

requirements-test.txt

@@ -0,0 +1,4 @@
+coverage==7.4.4
+pytest==8.1.1
+pytest-cov==4.1.0
+pytest-mock==3.12.0

requirements.txt

@@ -1,30 +1,34 @@
-coverage
-flake8
-flake8-quotes
-Flask
-Flask-WTF
-h3
-h3pandas
-itsdangerous
-jinja2
-keras
-matplotlib
-numpy
-pandas
-Pillow
-pipenv
-prison
-pytest
-pytest-cov
-pytest-mock
-python-magic
-pyyaml
-scikit_learn
-scipy
-six
-tensorflow
-tifffile
-torch
-umap_learn
-werkzeug
-WTForms
+flake8==7.0.0
+flake8-quotes==3.4.0
+Flask==3.0.2
+Flask-WTF==1.2.1
+h3==3.7.7
+h3pandas==0.2.6
+itsdangerous==2.1.2
+Jinja2==3.1.3
+keras==2.14.0;python_version>="3.11"
+keras==2.12.0;python_version=="3.8"
+matplotlib==3.8.3;python_version>="3.11"
+matplotlib==3.7.5;python_version=="3.8"
+numpy==1.26.4;python_version>="3.11"
+numpy==1.23.5;python_version=="3.8"
+pandas==2.1.2;python_version>="3.11"
+pandas==2.0.3;python_version=="3.8"
+pillow==10.2.0
+pipenv==2023.12.1
+prison==0.2.1
+python-magic==0.4.27
+PyYAML==6.0.1
+scikit-learn==1.4.1.post1;python_version>="3.11"
+scikit-learn==1.3.2;python_version=="3.8"
+scipy==1.12.0;python_version>="3.11"
+scipy==1.10.1;python_version=="3.8"
+six==1.16.0
+tensorflow==2.14.0;python_version>="3.11"
+tensorflow==2.12.0;python_version=="3.8"
+tifffile==2024.2.12;python_version>="3.11"
+tifffile==2023.7.10;python_version=="3.8"
+umap-learn==0.5.5
+uWSGI==2.0.24
+Werkzeug==3.0.1
+WTForms==3.1.2