Merge pull request #2 from Kitware/init-app

Initial Application Setup

LICENSE

@@ -174,28 +174,3 @@
       of your accepting any such warranty or additional liability.
 
    END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

MANIFEST.in

@@ -0,0 +1,2 @@
+# Include any data files such as templates or static assets in the source distribution
+graft bats_ai

NOTICE

@@ -0,0 +1,13 @@
+Copyright 2023 Kitware, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+See accompanying LICENSE file. You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

Procfile

@@ -0,0 +1,3 @@
+release: ./manage.py migrate
+web: gunicorn --bind 0.0.0.0:$PORT bats_ai.wsgi
+worker: REMAP_SIGTERM=SIGQUIT celery --app bats_ai.celery worker --loglevel INFO --without-heartbeat

README.md

@@ -1,2 +1,88 @@
-# BatAI
-Bat Detection and Classification using AI and Acoustics
+# bats-ai
+
+## Develop with Docker (recommended quickstart)
+This is the simplest configuration for developers to start with.
+
+### Initial Setup
+1. Run `docker compose run --rm django ./manage.py migrate`
+2. Run `docker compose run --rm django ./manage.py createsuperuser`
+   and follow the prompts to create your own user
+
+### Run Application
+1. Run `docker compose up`
+2. Access the site, starting at http://localhost:8000/admin/
+3. When finished, use `Ctrl+C`
+
+### Application Maintenance
+Occasionally, new package dependencies or schema changes will necessitate
+maintenance. To non-destructively update your development stack at any time:
+1. Run `docker compose pull`
+2. Run `docker compose build --pull --no-cache`
+3. Run `docker compose run --rm django ./manage.py migrate`
+
+## Develop Natively (advanced)
+This configuration still uses Docker to run attached services in the background,
+but allows developers to run Python code on their native system.
+
+### Initial Setup
+1. Run `docker compose -f ./docker-compose.yml up -d`
+2. Install Python 3.10
+3. Install
+   [`psycopg2` build prerequisites](https://www.psycopg.org/docs/install.html#build-prerequisites)
+4. Create and activate a new Python virtualenv
+5. Run `pip install -e .[dev]`
+6. Run `source ./dev/export-env.sh`
+7. Run `./manage.py migrate`
+8. Run `./manage.py createsuperuser` and follow the prompts to create your own user
+
+### Run Application
+1.  Ensure `docker compose -f ./docker-compose.yml up -d` is still active
+2. Run:
+   1. `source ./dev/export-env.sh`
+   2. `./manage.py runserver`
+3. Run in a separate terminal:
+   1. `source ./dev/export-env.sh`
+   2. `celery --app bats_ai.celery worker --loglevel INFO --without-heartbeat`
+4. When finished, run `docker compose stop`
+5. To destroy the stack and start fresh, run `docker compose down -v`
+
+## Remap Service Ports (optional)
+Attached services may be exposed to the host system via alternative ports. Developers who work
+on multiple software projects concurrently may find this helpful to avoid port conflicts.
+
+To do so, before running any `docker compose` commands, set any of the environment variables:
+* `DOCKER_POSTGRES_PORT`
+* `DOCKER_RABBITMQ_PORT`
+* `DOCKER_MINIO_PORT`
+
+The Django server must be informed about the changes:
+* When running the "Develop with Docker" configuration, override the environment variables:
+  * `DJANGO_MINIO_STORAGE_MEDIA_URL`, using the port from `DOCKER_MINIO_PORT`.
+* When running the "Develop Natively" configuration, override the environment variables:
+  * `DJANGO_DATABASE_URL`, using the port from `DOCKER_POSTGRES_PORT`
+  * `DJANGO_CELERY_BROKER_URL`, using the port from `DOCKER_RABBITMQ_PORT`
+  * `DJANGO_MINIO_STORAGE_ENDPOINT`, using the port from `DOCKER_MINIO_PORT`
+
+Since most of Django's environment variables contain additional content, use the values from
+the appropriate `dev/.env.docker-compose*` file as a baseline for overrides.
+
+## Testing
+### Initial Setup
+tox is used to execute all tests.
+tox is installed automatically with the `dev` package extra.
+
+When running the "Develop with Docker" configuration, all tox commands must be run as
+`docker-compose run --rm django tox`; extra arguments may also be appended to this form.
+
+### Running Tests
+Run `tox` to launch the full test suite.
+
+Individual test environments may be selectively run.
+This also allows additional options to be be added.
+Useful sub-commands include:
+* `tox -e lint`: Run only the style checks
+* `tox -e type`: Run only the type checks
+* `tox -e test`: Run only the pytest-driven tests
+
+To automatically reformat all code to comply with
+some (but not all) of the style checks, run `tox -e format`.

bats_ai/__init__.py

@@ -0,0 +1,7 @@
+# This project module is imported for us when Django starts. To ensure that Celery app is always
+# defined prior to any shared_task definitions (so those tasks will bind to the app), import
+# the Celery module here for side effects.
+from .celery import app as _celery_app  # noqa: F401
+
+# Do not import anything else from this file, to avoid interfering with the startup order of the
+# Celery app and Django's settings.

bats_ai/api.py

@@ -0,0 +1,4 @@
+from ninja import NinjaAPI
+
+api = NinjaAPI()
+

bats_ai/asgi.py

@@ -0,0 +1,11 @@
+import os
+
+import configurations.importer
+from django.core.asgi import get_asgi_application
+
+os.environ['DJANGO_SETTINGS_MODULE'] = 'bats_ai.settings'
+if not os.environ.get('DJANGO_CONFIGURATION'):
+    raise ValueError('The environment variable "DJANGO_CONFIGURATION" must be set.')
+configurations.importer.install()
+
+application = get_asgi_application()

bats_ai/celery.py

@@ -0,0 +1,16 @@
+import os
+
+from celery import Celery
+import configurations.importer
+
+os.environ['DJANGO_SETTINGS_MODULE'] = 'bats_ai.settings'
+if not os.environ.get('DJANGO_CONFIGURATION'):
+    raise ValueError('The environment variable "DJANGO_CONFIGURATION" must be set.')
+configurations.importer.install()
+
+# Using a string config_source means the worker doesn't have to serialize
+# the configuration object to child processes.
+app = Celery(config_source='django.conf:settings', namespace='CELERY')
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()

bats_ai/core/admin/__init__.py

@@ -0,0 +1,5 @@
+from .image import ImageAdmin
+
+__all__ = [
+    'ImageAdmin',
+]

bats_ai/core/admin/image.py

@@ -0,0 +1,41 @@
+from django.contrib import admin, messages
+from django.db.models import QuerySet
+from django.http import HttpRequest
+
+from bats_ai.core.models import Image
+from bats_ai.core.tasks import image_compute_checksum
+
+
+@admin.register(Image)
+class ImageAdmin(admin.ModelAdmin):
+    list_display = ['id', 'name', 'short_checksum', 'created', 'owner']
+    list_display_links = ['id', 'name']
+    list_filter = [
+        ('checksum', admin.EmptyFieldListFilter),
+        ('created', admin.DateFieldListFilter),
+        'owner__username',
+    ]
+    list_select_related = True
+    # list_select_related = ['owner']
+
+    search_fields = ['name']
+    actions = ['compute_checksum']
+
+    fields = ['name', 'blob', 'checksum', 'owner', 'created', 'modified']
+    autocomplete_fields = ['owner']
+    readonly_fields = ['checksum', 'created', 'modified']
+
+    @admin.display(
+        description='Checksum prefix',
+        empty_value='Not computed',
+        # Sorting by checksum also sorts the prefix values
+        ordering='checksum',
+    )
+    def short_checksum(self, image: Image):
+        return image.short_checksum
+
+    @admin.action(description='Recompute checksum')
+    def compute_checksum(self, request: HttpRequest, queryset: QuerySet):
+        for image in queryset:
+            image_compute_checksum.delay(image.pk)
+        self.message_user(request, f'{len(queryset)} images queued', messages.SUCCESS)

bats_ai/core/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class CoreConfig(AppConfig):
+    name = 'bats_ai.core'
+    verbose_name = 'bats-ai: Core'

bats_ai/core/management/commands/makeclient.py

@@ -0,0 +1,40 @@
+from django.contrib.auth.models import User
+import djclick as click
+from oauth2_provider.models import Application
+
+CLIENT_ID = 'HSJWFZ2cIpWQOvNyCXyStV9hiOd7DfWeBOCzo4pP'
+
+
+# create django oauth toolkit appliction (client)
+@click.option(
+    '--username',
+    type=click.STRING,
+    required=True,
+    help='superuser username for application creator',
+)
+@click.option('--uri', type=click.STRING, required=True, help='redirect uri for application')
+@click.command()
+def command(username, uri):
+    if Application.objects.filter(client_id=CLIENT_ID).exists():
+        click.echo('The client already exists. You can administer it from the admin console.')
+        return
+
+    if username:
+        user = User.objects.get(username=username)
+    else:
+        first_user = User.objects.first()
+        if first_user:
+            user = first_user
+
+    if user:
+        application = Application(
+            name='batsai-client',
+            client_id=CLIENT_ID,
+            client_secret='',
+            client_type='public',
+            redirect_uris=uri,
+            authorization_grant_type='authorization-code',
+            user=user,
+            skip_authorization=True,
+        )
+        application.save()

bats_ai/core/models/__init__.py

@@ -0,0 +1,6 @@
+
+from .image import Image
+
+__all__ = [
+    'Image',
+]

bats_ai/core/models/image.py

@@ -0,0 +1,25 @@
+from hashlib import sha512
+
+from django.contrib.auth.models import User
+from django.db import models
+from django_extensions.db.models import TimeStampedModel
+from s3_file_field import S3FileField
+
+
+class Image(TimeStampedModel, models.Model):
+    name = models.CharField(max_length=255)
+    blob = S3FileField()
+    checksum = models.CharField(max_length=128, blank=True, null=True)
+    owner = models.ForeignKey(User, on_delete=models.CASCADE)
+    # TimeStampedModel also provides "created" and "modified" fields
+
+    @property
+    def short_checksum(self) -> str | None:
+        return f'{self.checksum[:10]}' if self.checksum else None
+
+    def compute_checksum(self) -> None:
+        hasher = sha512()
+        with self.blob.open() as blob:
+            for chunk in blob.chunks():
+                hasher.update(chunk)
+        self.checksum = hasher.hexdigest()

bats_ai/core/rest/__init__.py

@@ -0,0 +1,21 @@
+from .acoustic_batch import AcousticBatchViewSet
+from .acoustic_file import AcousticFileViewSet
+from .acoustic_file_batch import AcousticFileBatchViewSet
+from .image import ImageViewSet
+from .projects import ProjectViewSet
+from .species import SpeciesViewSet
+from .spectrogram import Spectrogram
+from .survey_events import SurveyEventViewSet
+from .surveys import SurveyViewSet
+
+__all__ = [
+    'ImageViewSet',
+    'ProjectViewSet',
+    'SurveyViewSet',
+    'SurveyEventViewSet',
+    'AcousticBatchViewSet',
+    'AcousticFileViewSet',
+    'AcousticFileBatchViewSet',
+    'SpeciesViewSet',
+    'Spectrogram',
+]

bats_ai/core/rest/image.py

@@ -0,0 +1,45 @@
+from django.http import HttpResponseRedirect
+from django_filters import rest_framework as filters
+from rest_framework import serializers, status
+from rest_framework.decorators import action
+from rest_framework.pagination import PageNumberPagination
+from rest_framework.permissions import IsAuthenticatedOrReadOnly
+from rest_framework.response import Response
+from rest_framework.viewsets import ReadOnlyModelViewSet
+
+from bats_ai.core.models import Image
+from bats_ai.core.rest.user import UserSerializer
+from bats_ai.core.tasks import image_compute_checksum
+
+
+class ImageSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = Image
+        fields = ['id', 'name', 'checksum', 'created', 'owner']
+        read_only_fields = ['checksum', 'created']
+
+    owner = UserSerializer()
+
+
+class ImageViewSet(ReadOnlyModelViewSet):
+    queryset = Image.objects.all()
+
+    permission_classes = [IsAuthenticatedOrReadOnly]
+    serializer_class = ImageSerializer
+
+    filter_backends = [filters.DjangoFilterBackend]
+    filterset_fields = ['name', 'checksum']
+
+    pagination_class = PageNumberPagination
+
+    @action(detail=True, methods=['get'])
+    def download(self, request, pk=None):
+        image = self.get_object()
+        return HttpResponseRedirect(image.blob.url)
+
+    @action(detail=True, methods=['post'])
+    def compute(self, request, pk=None):
+        # Ensure that the image exists, so a non-existent pk isn't dispatched
+        image = self.get_object()
+        image_compute_checksum.delay(image.pk)
+        return Response('', status=status.HTTP_202_ACCEPTED)

bats_ai/core/tasks.py

@@ -0,0 +1,10 @@
+from celery import shared_task
+
+from bats_ai.core.models import Image
+
+
+@shared_task
+def image_compute_checksum(image_id: int):
+    image = Image.objects.get(pk=image_id)
+    image.compute_checksum()
+    image.save()