Guano Metadata Extraction (#95)

* initial metadata extraction

* add models to recording

* client side guano metadata

* add fron-end tool for details

* linting

* batch uploading guano metadata

* client linting

* support for additional time formats

bats_ai/api.py

@@ -3,7 +3,7 @@
 from ninja import NinjaAPI
 from oauth2_provider.models import AccessToken
 
-from bats_ai.core.views import GRTSCellsRouter, RecordingRouter, SpeciesRouter
+from bats_ai.core.views import GRTSCellsRouter, GuanoMetadataRouter, RecordingRouter, SpeciesRouter
 
 logger = logging.getLogger(__name__)
 
@@ -27,3 +27,4 @@ def global_auth(request):
 api.add_router('/recording/', RecordingRouter)
 api.add_router('/species/', SpeciesRouter)
 api.add_router('/grts/', GRTSCellsRouter)
+api.add_router('/guano/', GuanoMetadataRouter)

bats_ai/core/admin/recording.py

@@ -24,6 +24,13 @@ class RecordingAdmin(admin.ModelAdmin):
         'recording_location',
         'grts_cell_id',
         'grts_cell',
+        'site_name',
+        'detector',
+        'software',
+        'species_list',
+        'unusual_occurrences',
+        'get_computed_species',
+        'get_official_species',
     ]
     list_select_related = True
     # list_select_related = ['owner']
@@ -34,6 +41,12 @@ class RecordingAdmin(admin.ModelAdmin):
     autocomplete_fields = ['owner']
     readonly_fields = ['created', 'modified']
 
+    def get_official_species(self, instance):
+        return [species.species_code_6 for species in instance.official_species.all()]
+
+    def get_computed_species(self, instance):
+        return [species.species_code_6 for species in instance.computed_species.all()]
+
     @admin.display(
         description='Spectrogram',
         empty_value='Not computed',

bats_ai/core/migrations/0010_recording_computed_species_recording_detector_and_more.py

@@ -0,0 +1,51 @@
+# Generated by Django 4.1.13 on 2024-04-03 13:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('core', '0009_annotations_type'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='recording',
+            name='computed_species',
+            field=models.ManyToManyField(
+                related_name='recording_computed_species', to='core.species'
+            ),
+        ),
+        migrations.AddField(
+            model_name='recording',
+            name='detector',
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='recording',
+            name='official_species',
+            field=models.ManyToManyField(
+                related_name='recording_official_species', to='core.species'
+            ),
+        ),
+        migrations.AddField(
+            model_name='recording',
+            name='site_name',
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='recording',
+            name='software',
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='recording',
+            name='species_list',
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='recording',
+            name='unusual_occurrences',
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

bats_ai/core/models/recording.py

@@ -2,6 +2,8 @@
 from django.contrib.gis.db import models
 from django_extensions.db.models import TimeStampedModel
 
+from .species import Species
+
 
 # TimeStampedModel also provides "created" and "modified" fields
 class Recording(TimeStampedModel, models.Model):
@@ -16,6 +18,17 @@ class Recording(TimeStampedModel, models.Model):
     grts_cell_id = models.IntegerField(blank=True, null=True)
     grts_cell = models.IntegerField(blank=True, null=True)
     public = models.BooleanField(default=False)
+    software = models.TextField(blank=True, null=True)
+    detector = models.TextField(blank=True, null=True)
+    species_list = models.TextField(blank=True, null=True)
+    site_name = models.TextField(blank=True, null=True)
+    computed_species = models.ManyToManyField(
+        Species, related_name='recording_computed_species'
+    )  # species from a computed sense
+    official_species = models.ManyToManyField(
+        Species, related_name='recording_official_species'
+    )  # species that are detemrined by the owner or from annotations as official species list
+    unusual_occurrences = models.TextField(blank=True, null=True)
 
     @property
     def has_spectrogram(self):

bats_ai/core/views/__init__.py

@@ -1,5 +1,6 @@
 from .annotations import router as AnnotationRouter
 from .grts_cells import router as GRTSCellsRouter
+from .guanometadata import router as GuanoMetadataRouter
 from .recording import router as RecordingRouter
 from .species import router as SpeciesRouter
 from .temporal_annotations import router as TemporalAnnotationRouter
@@ -10,4 +11,5 @@
     'AnnotationRouter',
     'TemporalAnnotationRouter',
     'GRTSCellsRouter',
+    'GuanoMetadataRouter',
 ]

bats_ai/core/views/guanometadata.py

@@ -0,0 +1,88 @@
+from datetime import datetime
+import logging
+
+from django.http import HttpRequest, JsonResponse
+from guano import GuanoFile
+from ninja import File, Schema
+from ninja.files import UploadedFile
+from ninja.pagination import RouterPaginated
+
+router = RouterPaginated()
+logger = logging.getLogger(__name__)
+
+
+class GuanoMetadataSchema(Schema):
+    nabat_grid_cell_grts_id: str | None = None
+    nabat_latitude: float | None = None
+    nabat_longitude: float | None = None
+    nabat_site_name: str | None = None
+    nabat_activation_start_time: datetime | None = None
+    nabat_activation_end_time: datetime | None = None
+    nabat_software_type: str | None = None
+    nabat_species_list: list[str] | None = None
+    nabat_comments: str | None = None
+    nabat_detector_type: str | None = None
+    nabat_unusual_occurrences: str | None = None
+
+
+router = RouterPaginated()
+
+
+@router.post('/')
+def default_data(
+    request: HttpRequest,
+    audio_file: File[UploadedFile],
+):
+    try:
+        # Read GUANO metadata from the file name provided
+        gfile = GuanoFile(audio_file.file.name)
+
+        # Extract required NABat fields
+        nabat_fields = {
+            'nabat_grid_cell_grts_id': gfile.get('NABat|Grid Cell GRTS ID', None),
+            'nabat_latitude': (gfile.get('NABat|Latitude', None)),
+            'nabat_longitude': (gfile.get('NABat|Longitude', None)),
+            'nabat_site_name': gfile.get('NABat|Site Name', None),
+        }
+
+        # Extract additional fields with conditionals
+        additional_fields = {
+            'nabat_activation_start_time': parse_datetime(
+                gfile.get('NABat|Activation start time', None)
+            )
+            if 'NABat|Activation start time' in gfile
+            else None,
+            'nabat_activation_end_time': parse_datetime(
+                gfile.get('NABat|Activation end time', None)
+            )
+            if 'NABat|Activation end time' in gfile
+            else None,
+            'nabat_software_type': gfile.get('NABat|Software type', None),
+            'nabat_species_list': gfile.get('NABat|Species List', '').split(','),
+            'nabat_comments': gfile.get('NABat|Comments', None),
+            'nabat_detector_type': gfile.get('NABat|Detector type', None),
+            'nabat_unusual_occurrences': gfile.get('NABat|Unusual occurrences', ''),
+        }
+
+        # Combine all extracted fields
+        metadata = {**nabat_fields, **additional_fields}
+
+        return JsonResponse(metadata, safe=False)
+
+    except Exception as e:
+        return JsonResponse({'error': str(e)}, status=500)
+
+
+def parse_datetime(datetime_str):
+    if datetime_str:
+        try:
+            # Try parsing using the custom format
+            return datetime.strptime(datetime_str, '%Y%m%dT%H%M%S')
+        except ValueError:
+            try:
+                # Try parsing using ISO format
+                return datetime.fromisoformat(datetime_str)
+            except ValueError:
+                # If both formats fail, return None or handle the error accordingly
+                return None
+    return None

bats_ai/core/views/recording.py

@@ -47,6 +47,11 @@ class RecordingUploadSchema(Schema):
     longitude: float = None
     gridCellId: int = None
     publicVal: bool = None
+    site_name: str = None
+    software: str = None
+    detector: str = None
+    species_list: str = None
+    unusual_occurrences: str = None
 
 
 class AnnotationSchema(Schema):
@@ -109,7 +114,13 @@ def create_recording(
         recording_location=point,
         public=publicVal,
         comments=payload.comments,
+        detector=payload.detector,
+        software=payload.software,
+        site_name=payload.site_name,
+        species_list=payload.species_list,
+        unusual_occurrences=payload.unusual_occurrences,
     )
+
     recording.save()
     # Start generating recording as soon as created
     # this creates the spectrogram during the upload so it is available immediately afterwards
@@ -142,6 +153,16 @@ def update_recording(request: HttpRequest, id: int, recording_data: RecordingUpl
     if recording_data.latitude and recording_data.longitude:
         point = Point(recording_data.longitude, recording_data.latitude)
         recording.recording_location = point
+    if recording_data.detector:
+        recording.detector = recording_data.detector
+    if recording_data.software:
+        recording.software = recording_data.software
+    if recording_data.site_name:
+        recording.site_name = recording_data.site_name
+    if recording_data.species_list:
+        recording.species_list = recording_data.species_list
+    if recording_data.unusual_occurrences:
+        recording.unusual_occurrences = recording_data.unusual_occurrences
 
     recording.save()
 

bats_ai/settings.py

@@ -21,6 +21,10 @@ class BatsAiMixin(ConfigMixin):
 
     BASE_DIR = Path(__file__).resolve(strict=True).parent.parent
 
+    FILE_UPLOAD_HANDLERS = [
+        'django.core.files.uploadhandler.TemporaryFileUploadHandler',
+    ]
+
     @staticmethod
     def mutate_configuration(configuration: ComposedConfiguration) -> None:
         # Install local apps first, to ensure any overridden resources are found first