diff --git a/.gitattributes b/.gitattributes index 84fe760..f8770d0 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ assets/example.wav filter=lfs diff=lfs merge=lfs -text +assets/model.mobilenet.onnx filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index 78d3c39..638a871 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,10 @@ /**/*.shp /**/*.shx /**/*.csv +models/datasets/ +models/spectrograms/ +models/ignore/ +models/*.jpg +models/*.pkl +temp*.jpg +temp*.png diff --git a/README.md b/README.md index dda410f..112b5a8 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,9 @@ This is the simplest configuration for developers to start with. ### Run Vue Frontend -1. Run `npm install` -2. Run `npm run dev` +1. Run `cd client/` +2. Run `npm install` +3. Run `npm run dev` ### Run Application diff --git a/assets/model.mobilenet.onnx b/assets/model.mobilenet.onnx new file mode 100644 index 0000000..898cca5 --- /dev/null +++ b/assets/model.mobilenet.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6e36494aa6457f8d93bb882e90fe7bf2100484fdaa2085d15856217ba1fcbf1 +size 12135629 diff --git a/bats_ai/core/admin/__init__.py b/bats_ai/core/admin/__init__.py index 48b05ad..8f782b8 100644 --- a/bats_ai/core/admin/__init__.py +++ b/bats_ai/core/admin/__init__.py @@ -1,4 +1,5 @@ from .annotations import AnnotationsAdmin +from .compressed_spectrogram import CompressedSpectrogramAdmin from .grts_cells import GRTSCellsAdmin from .image import ImageAdmin from .recording import RecordingAdmin @@ -14,4 +15,5 @@ 'TemporalAnnotationsAdmin', 'SpeciesAdmin', 'GRTSCellsAdmin', + 'CompressedSpectrogramAdmin', ] diff --git a/bats_ai/core/admin/compressed_spectrogram.py b/bats_ai/core/admin/compressed_spectrogram.py new file mode 100644 index 0000000..a8ce90c --- /dev/null +++ b/bats_ai/core/admin/compressed_spectrogram.py @@ -0,0 +1,30 @@ +from django.contrib import admin + +from bats_ai.core.models import CompressedSpectrogram + + +@admin.register(CompressedSpectrogram) +class CompressedSpectrogramAdmin(admin.ModelAdmin): + list_display = [ + 'pk', + 'recording', + 'spectrogram', + 'image_file', + 'length', + 'widths', + 'starts', + 'stops', + ] + list_display_links = ['pk', 'recording', 'spectrogram'] + list_select_related = True + autocomplete_fields = ['recording'] + readonly_fields = [ + 'recording', + 'spectrogram', + 'image_file', + 'created', + 'modified', + 'widths', + 'starts', + 'stops', + ] diff --git a/bats_ai/core/admin/recording.py b/bats_ai/core/admin/recording.py index 8cc217d..bb2d5d3 100644 --- a/bats_ai/core/admin/recording.py +++ b/bats_ai/core/admin/recording.py @@ -64,7 +64,6 @@ def spectrogram_status(self, recording: Recording): def compute_spectrograms(self, request: HttpRequest, queryset: QuerySet): counter = 0 for recording in queryset: - if not recording.has_spectrogram: - recording_compute_spectrogram.delay(recording.pk) - counter += 1 + recording_compute_spectrogram.delay(recording.pk) + counter += 1 self.message_user(request, f'{counter} recordings queued', messages.SUCCESS) diff --git a/bats_ai/core/admin/spectrogram.py b/bats_ai/core/admin/spectrogram.py index 2586ebb..6f2f601 100644 --- a/bats_ai/core/admin/spectrogram.py +++ b/bats_ai/core/admin/spectrogram.py @@ -8,6 +8,7 @@ class SpectrogramAdmin(admin.ModelAdmin): list_display = [ 'pk', 'recording', + 'colormap', 'image_file', 'width', 'height', @@ -20,6 +21,7 @@ class SpectrogramAdmin(admin.ModelAdmin): autocomplete_fields = ['recording'] readonly_fields = [ 'recording', + 'colormap', 'image_file', 'created', 'modified', diff --git a/bats_ai/core/migrations/0009_annotations_type.py b/bats_ai/core/migrations/0009_annotations_type.py deleted file mode 100644 index 6091cd7..0000000 --- a/bats_ai/core/migrations/0009_annotations_type.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 4.1.13 on 2024-03-22 17:23 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ('core', '0008_grtscells_recording_recorded_time'), - ] - - operations = [ - migrations.AddField( - model_name='annotations', - name='type', - field=models.TextField(blank=True, null=True), - ), - ] diff --git a/bats_ai/core/migrations/0010_recording_computed_species_recording_detector_and_more.py b/bats_ai/core/migrations/0009_annotations_type_recording_computed_species_and_more.py similarity index 76% rename from bats_ai/core/migrations/0010_recording_computed_species_recording_detector_and_more.py rename to bats_ai/core/migrations/0009_annotations_type_recording_computed_species_and_more.py index 1819063..ce429a8 100644 --- a/bats_ai/core/migrations/0010_recording_computed_species_recording_detector_and_more.py +++ b/bats_ai/core/migrations/0009_annotations_type_recording_computed_species_and_more.py @@ -1,14 +1,19 @@ -# Generated by Django 4.1.13 on 2024-04-03 13:07 +# Generated by Django 4.1.13 on 2024-04-11 13:06 from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('core', '0009_annotations_type'), + ('core', '0008_grtscells_recording_recorded_time'), ] operations = [ + migrations.AddField( + model_name='annotations', + name='type', + field=models.TextField(blank=True, null=True), + ), migrations.AddField( model_name='recording', name='computed_species', @@ -48,4 +53,9 @@ class Migration(migrations.Migration): name='unusual_occurrences', field=models.TextField(blank=True, null=True), ), + migrations.AddField( + model_name='spectrogram', + name='colormap', + field=models.CharField(max_length=20, null=True), + ), ] diff --git a/bats_ai/core/migrations/0010_compressedspectrogram.py b/bats_ai/core/migrations/0010_compressedspectrogram.py new file mode 100644 index 0000000..28fe4fb --- /dev/null +++ b/bats_ai/core/migrations/0010_compressedspectrogram.py @@ -0,0 +1,84 @@ +# Generated by Django 4.1.13 on 2024-04-19 13:55 + +import django.contrib.postgres.fields +from django.db import migrations, models +import django.db.models.deletion +import django_extensions.db.fields + + +class Migration(migrations.Migration): + dependencies = [ + ('core', '0009_annotations_type_recording_computed_species_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='CompressedSpectrogram', + fields=[ + ( + 'id', + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name='ID' + ), + ), + ( + 'created', + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name='created' + ), + ), + ( + 'modified', + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name='modified' + ), + ), + ('image_file', models.FileField(upload_to='')), + ('length', models.IntegerField()), + ( + 'starts', + django.contrib.postgres.fields.ArrayField( + base_field=django.contrib.postgres.fields.ArrayField( + base_field=models.IntegerField(), size=None + ), + size=None, + ), + ), + ( + 'stops', + django.contrib.postgres.fields.ArrayField( + base_field=django.contrib.postgres.fields.ArrayField( + base_field=models.IntegerField(), size=None + ), + size=None, + ), + ), + ( + 'widths', + django.contrib.postgres.fields.ArrayField( + base_field=django.contrib.postgres.fields.ArrayField( + base_field=models.IntegerField(), size=None + ), + size=None, + ), + ), + ('cache_invalidated', models.BooleanField(default=True)), + ( + 'recording', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to='core.recording' + ), + ), + ( + 'spectrogram', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to='core.spectrogram' + ), + ), + ], + options={ + 'get_latest_by': 'modified', + 'abstract': False, + }, + ), + ] diff --git a/bats_ai/core/models/__init__.py b/bats_ai/core/models/__init__.py index 553e7cb..36e2f0f 100644 --- a/bats_ai/core/models/__init__.py +++ b/bats_ai/core/models/__init__.py @@ -1,7 +1,8 @@ from .annotations import Annotations +from .compressed_spectrogram import CompressedSpectrogram from .grts_cells import GRTSCells from .image import Image -from .recording import Recording +from .recording import Recording, colormap from .recording_annotation_status import RecordingAnnotationStatus from .species import Species from .spectrogram import Spectrogram @@ -16,4 +17,6 @@ 'Spectrogram', 'TemporalAnnotations', 'GRTSCells', + 'colormap', + 'CompressedSpectrogram', ] diff --git a/bats_ai/core/models/compressed_spectrogram.py b/bats_ai/core/models/compressed_spectrogram.py new file mode 100644 index 0000000..5dd57c1 --- /dev/null +++ b/bats_ai/core/models/compressed_spectrogram.py @@ -0,0 +1,112 @@ +from PIL import Image +import cv2 +from django.contrib.postgres.fields import ArrayField +from django.core.files.storage import default_storage +from django.db import models +from django.dispatch import receiver +from django_extensions.db.models import TimeStampedModel +import numpy as np + +from .recording import Recording +from .spectrogram import Spectrogram + +FREQ_MIN = 5e3 +FREQ_MAX = 120e3 +FREQ_PAD = 2e3 + + +# TimeStampedModel also provides "created" and "modified" fields +class CompressedSpectrogram(TimeStampedModel, models.Model): + recording = models.ForeignKey(Recording, on_delete=models.CASCADE) + spectrogram = models.ForeignKey(Spectrogram, on_delete=models.CASCADE) + image_file = models.FileField() + length = models.IntegerField() + starts = ArrayField(ArrayField(models.IntegerField())) + stops = ArrayField(ArrayField(models.IntegerField())) + widths = ArrayField(ArrayField(models.IntegerField())) + cache_invalidated = models.BooleanField(default=True) + + @property + def image_url(self): + return default_storage.url(self.image_file.name) + + def predict(self): + import json + import os + + import onnx + import onnxruntime as ort + import tqdm + + img = Image.open(self.image_file) + + relative = ('..',) * 4 + asset_path = os.path.abspath(os.path.join(__file__, *relative, 'assets')) + + onnx_filename = os.path.join(asset_path, 'model.mobilenet.onnx') + assert os.path.exists(onnx_filename) + + session = ort.InferenceSession( + onnx_filename, + providers=[ + ( + 'CUDAExecutionProvider', + { + 'cudnn_conv_use_max_workspace': '1', + 'device_id': 0, + 'cudnn_conv_algo_search': 'HEURISTIC', + }, + ), + 'CPUExecutionProvider', + ], + ) + + img = np.array(img) + + h, w, c = img.shape + ratio_y = 224 / h + ratio_x = ratio_y * 0.5 + raw = cv2.resize(img, None, fx=ratio_x, fy=ratio_y, interpolation=cv2.INTER_LANCZOS4) + + h, w, c = raw.shape + if w <= h: + canvas = np.zeros((h, h + 1, 3), dtype=raw.dtype) + canvas[:, :w, :] = raw + raw = canvas + h, w, c = raw.shape + + inputs_ = [] + for index in range(0, w - h, 100): + inputs_.append(raw[:, index : index + h, :]) + inputs_.append(raw[:, -h:, :]) + inputs_ = np.array(inputs_) + + chunksize = 1 + chunks = np.array_split(inputs_, np.arange(chunksize, len(inputs_), chunksize)) + outputs = [] + for chunk in tqdm.tqdm(chunks, desc='Inference'): + outputs_ = session.run( + None, + {'input': chunk}, + ) + outputs.append(outputs_[0]) + outputs = np.vstack(outputs) + outputs = outputs.mean(axis=0) + + model = onnx.load(onnx_filename) + mapping = json.loads(model.metadata_props[0].value) + labels = [mapping['forward'][str(index)] for index in range(len(mapping['forward']))] + + prediction = np.argmax(outputs) + label = labels[prediction] + score = outputs[prediction] + + confs = dict(zip(labels, outputs)) + + return label, score, confs + + +@receiver(models.signals.pre_delete, sender=Spectrogram) +def delete_content(sender, instance, **kwargs): + if instance.image_file: + instance.image_file.delete(save=False) diff --git a/bats_ai/core/models/recording.py b/bats_ai/core/models/recording.py index 6d81036..4eb3b42 100644 --- a/bats_ai/core/models/recording.py +++ b/bats_ai/core/models/recording.py @@ -1,9 +1,34 @@ +import logging + from django.contrib.auth.models import User from django.contrib.gis.db import models +from django.dispatch import receiver from django_extensions.db.models import TimeStampedModel from .species import Species +logger = logging.getLogger(__name__) + + +COLORMAP = None + + +class colormap: + def __init__(self, colormap=None): + self.colormap = colormap + self.previous = None + + def __enter__(self): + global COLORMAP + + self.previous = COLORMAP + COLORMAP = self.colormap + + def __exit__(self, exc_type, exc_value, exc_tb): + global COLORMAP + + COLORMAP = self.previous + # TimeStampedModel also provides "created" and "modified" fields class Recording(TimeStampedModel, models.Model): @@ -38,22 +63,22 @@ def has_spectrogram(self): def spectrograms(self): from bats_ai.core.models import Spectrogram - query = Spectrogram.objects.filter(recording=self).order_by('-created') + query = Spectrogram.objects.filter(recording=self, colormap=COLORMAP).order_by('-created') return query.all() @property def spectrogram(self): - from bats_ai.core.models import Spectrogram + pass spectrograms = self.spectrograms - if len(spectrograms) == 0: - Spectrogram.generate(self) - - spectrograms = self.spectrograms - assert len(spectrograms) == 1 - assert len(spectrograms) >= 1 spectrogram = spectrograms[0] # most recently created return spectrogram + + +@receiver(models.signals.pre_delete, sender=Recording) +def delete_content(sender, instance, **kwargs): + if instance.audio_file: + instance.audio_file.delete(save=False) diff --git a/bats_ai/core/models/spectrogram.py b/bats_ai/core/models/spectrogram.py index 9ee4ab6..eaedefb 100644 --- a/bats_ai/core/models/spectrogram.py +++ b/bats_ai/core/models/spectrogram.py @@ -1,24 +1,31 @@ import base64 import io +import logging import math from PIL import Image import cv2 from django.core.files import File +from django.core.files.storage import default_storage from django.db import models from django.db.models.fields.files import FieldFile +from django.dispatch import receiver from django_extensions.db.models import TimeStampedModel import librosa import matplotlib.pyplot as plt import numpy as np -import scipy +import tqdm -from bats_ai.core.models import Recording +from .recording import Recording + +logger = logging.getLogger(__name__) FREQ_MIN = 5e3 FREQ_MAX = 120e3 FREQ_PAD = 2e3 +COLORMAP_ALLOWED = [None, 'gist_yarg', 'turbo'] + # TimeStampedModel also provides "created" and "modified" fields class Spectrogram(TimeStampedModel, models.Model): @@ -29,9 +36,13 @@ class Spectrogram(TimeStampedModel, models.Model): duration = models.IntegerField() # milliseconds frequency_min = models.IntegerField() # hz frequency_max = models.IntegerField() # hz + colormap = models.CharField(max_length=20, blank=False, null=True) @classmethod - def generate(cls, recording): + def generate(cls, recording, colormap=None, dpi=520): + """ + Ref: https://matplotlib.org/stable/users/explain/colors/colormaps.html + """ wav_file = recording.audio_file try: if isinstance(wav_file, FieldFile): @@ -45,13 +56,37 @@ def generate(cls, recording): print(e) return None + # Helpful aliases + size_mod = 1 + high_res = False + inference = False + + if colormap in ['inference']: + colormap = None + dpi = 300 + size_mod = 0 + inference = True + if colormap in ['none', 'default', 'dark']: + colormap = None + if colormap in ['light']: + colormap = 'gist_yarg' + if colormap in ['heatmap']: + colormap = 'turbo' + high_res = True + + # Supported colormaps + if colormap not in COLORMAP_ALLOWED: + logger.warning(f'Substituted requested {colormap} colormap to default') + logger.warning('See COLORMAP_ALLOWED') + colormap = None + # Function to take a signal and return a spectrogram. size = int(0.001 * sr) # 1.0ms resolution - size = 2 ** math.ceil(math.log(size, 2)) + size = 2 ** (math.ceil(math.log(size, 2)) + size_mod) hop_length = int(size / 4) # Short-time Fourier Transform - window = librosa.stft(sig, n_fft=size, window='hamming') + window = librosa.stft(sig, n_fft=size, hop_length=hop_length, window='hamming') # Calculating and processing data for the spectrogram. window = np.abs(window) ** 2 @@ -79,14 +114,13 @@ def generate(cls, recording): freq_high = int(FREQ_MAX + FREQ_PAD) vmin = window.min() vmax = window.max() - dpi = 300 - chunksize = int(5e3) + chunksize = int(2e3) arange = np.arange(chunksize, window.shape[1], chunksize) chunks = np.array_split(window, arange, axis=1) imgs = [] - for chunk in chunks: + for chunk in tqdm.tqdm(chunks): h, w = chunk.shape alpha = 3 figsize = (int(math.ceil(w / h)) * alpha + 1, alpha) @@ -94,17 +128,22 @@ def generate(cls, recording): ax = plt.axes() plt.margins(0) + kwargs = { + 'sr': sr, + 'n_fft': size, + 'hop_length': hop_length, + 'x_axis': 's', + 'y_axis': 'fft', + 'ax': ax, + 'vmin': vmin, + 'vmax': vmax, + } + # Plot - librosa.display.specshow( - chunk, - sr=sr, - hop_length=hop_length, - x_axis='s', - y_axis='linear', - ax=ax, - vmin=vmin, - vmax=vmax, - ) + if colormap is None: + librosa.display.specshow(chunk, **kwargs) + else: + librosa.display.specshow(chunk, cmap=colormap, **kwargs) ax.set_ylim(freq_low, freq_high) ax.axis('off') @@ -126,24 +165,61 @@ def generate(cls, recording): imgs.append(img) + if inference: + w_ = int(4.0 * duration * 1e3) + h_ = int(dpi) + else: + w_ = int(8.0 * duration * 1e3) + h_ = 1200 + img = np.hstack(imgs) - img = Image.fromarray(img, 'RGB') + img = cv2.resize(img, (w_, h_), interpolation=cv2.INTER_LANCZOS4) - w, h = img.size - # ratio = dpi / h - # w_ = int(round(w * ratio)) - w_ = int(4.0 * duration * 1e3) - h_ = int(dpi) - img = img.resize((w_, h_), resample=Image.Resampling.LANCZOS) - w, h = img.size + if high_res: + img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) + + noise = 0.1 + img = img.astype(np.float32) + img -= img.min() + img /= img.max() + img *= 255.0 + img /= 1.0 - noise + img[img < 0] = 0 + img[img > 255] = 255 + img = 255.0 - img # invert + + img = cv2.blur(img, (9, 9)) + img = cv2.resize(img, None, fx=2.0, fy=2.0, interpolation=cv2.INTER_LANCZOS4) + img = cv2.blur(img, (9, 9)) + + img -= img.min() + img /= img.max() + img *= 255.0 + + mask = (img > 255 * noise).astype(np.float32) + mask = cv2.blur(mask, (5, 5)) + + img[img < 0] = 0 + img[img > 255] = 255 + img = np.around(img).astype(np.uint8) + img = cv2.applyColorMap(img, cv2.COLORMAP_TURBO) + + img = img.astype(np.float32) + img *= mask.reshape(*mask.shape, 1) + img[img < 0] = 0 + img[img > 255] = 255 + img = np.around(img).astype(np.uint8) - # img.save('temp.jpg') + # cv2.imwrite('temp.png', img) + + img = Image.fromarray(img, 'RGB') + w, h = img.size buf = io.BytesIO() img.save(buf, format='JPEG', quality=80) buf.seek(0) - name = 'spectrogram.jpg' + name = f'{recording.pk}_{colormap}_spectrogram.jpg' image_file = File(buf, name=name) spectrogram = cls( @@ -154,128 +230,10 @@ def generate(cls, recording): duration=math.ceil(duration * 1e3), frequency_min=freq_low, frequency_max=freq_high, + colormap=colormap, ) spectrogram.save() - - @property - def compressed(self): - img = self.image_np - - threshold = 0.5 - while True: - canvas = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) - canvas = canvas.astype(np.float32) - - amplitude = canvas.max(axis=0) - amplitude -= amplitude.min() - amplitude /= amplitude.max() - amplitude[amplitude < threshold] = 0.0 - amplitude[amplitude > 0] = 1.0 - amplitude = amplitude.reshape(1, -1) - - canvas -= canvas.min() - canvas /= canvas.max() - canvas *= 255.0 - canvas *= amplitude - canvas = np.around(canvas).astype(np.uint8) - - mask = canvas.max(axis=0) - mask = scipy.signal.medfilt(mask, 3) - mask[0] = 0 - mask[-1] = 0 - starts = [] - stops = [] - for index in range(1, len(mask) - 1): - value_pre = mask[index - 1] - value = mask[index] - value_post = mask[index + 1] - if value != 0: - if value_pre == 0: - starts.append(index) - if value_post == 0: - stops.append(index) - assert len(starts) == len(stops) - - starts = [val - 40 for val in starts] # 10 ms buffer - stops = [val + 40 for val in stops] # 10 ms buffer - ranges = list(zip(starts, stops)) - - while True: - found = False - merged = [] - index = 0 - while index < len(ranges) - 1: - start1, stop1 = ranges[index] - start2, stop2 = ranges[index + 1] - - start1 = min(max(start1, 0), len(mask)) - start2 = min(max(start2, 0), len(mask)) - stop1 = min(max(stop1, 0), len(mask)) - stop2 = min(max(stop2, 0), len(mask)) - - if stop1 >= start2: - found = True - merged.append((start1, stop2)) - index += 2 - else: - merged.append((start1, stop1)) - index += 1 - if index == len(ranges) - 1: - merged.append((start2, stop2)) - ranges = merged - if not found: - for index in range(1, len(ranges)): - start1, stop1 = ranges[index - 1] - start2, stop2 = ranges[index] - assert start1 < stop1 - assert start2 < stop2 - assert start1 < start2 - assert stop1 < stop2 - assert stop1 < start2 - break - - segments = [] - starts_ = [] - stops_ = [] - domain = img.shape[1] - widths = [] - total_width = 0 - for start, stop in ranges: - segment = img[:, start:stop] - segments.append(segment) - - starts_.append(int(round(self.duration * (start / domain)))) - stops_.append(int(round(self.duration * (stop / domain)))) - widths.append(stop - start) - total_width += stop - start - - # buffer = np.zeros((len(img), 20, 3), dtype=img.dtype) - # segments.append(buffer) - # segments = segments[:-1] - - if len(segments) > 0: - break - - threshold -= 0.05 - if threshold < 0: - segments = None - break - - if segments is None: - canvas = img.copy() - else: - canvas = np.hstack(segments) - - canvas = Image.fromarray(canvas, 'RGB') - - # canvas.save('temp.jpg') - - buf = io.BytesIO() - canvas.save(buf, format='JPEG', quality=80) - buf.seek(0) - img_base64 = base64.b64encode(buf.getvalue()).decode('utf-8') - - return img_base64, starts_, stops_, widths, total_width + return spectrogram.pk @property def image_np(self): @@ -296,3 +254,13 @@ def base64(self): img_base64 = base64.b64encode(img).decode('utf-8') return img_base64 + + @property + def image_url(self): + return default_storage.url(self.image_file.name) + + +@receiver(models.signals.pre_delete, sender=Spectrogram) +def delete_content(sender, instance, **kwargs): + if instance.image_file: + instance.image_file.delete(save=False) diff --git a/bats_ai/core/rest/__init__.py b/bats_ai/core/rest/__init__.py index a3bdd5b..620d9f2 100644 --- a/bats_ai/core/rest/__init__.py +++ b/bats_ai/core/rest/__init__.py @@ -1,5 +1,13 @@ -from .image import ImageViewSet +from rest_framework import routers + +from .compressed_spectrogram import CompressedSpectrogramViewSet +from .spectrogram import SpectrogramViewSet __all__ = [ - 'ImageViewSet', + 'SpectrogramViewSet', + 'CompressedSpectrogramViewSet', ] + +rest = routers.SimpleRouter() +rest.register(r'spectrograms', SpectrogramViewSet) +rest.register(r'compressed_spectrograms', CompressedSpectrogramViewSet) diff --git a/bats_ai/core/rest/compressed_spectrogram.py b/bats_ai/core/rest/compressed_spectrogram.py new file mode 100644 index 0000000..3cdaf10 --- /dev/null +++ b/bats_ai/core/rest/compressed_spectrogram.py @@ -0,0 +1,21 @@ +from django_large_image.rest import LargeImageFileDetailMixin +from rest_framework import mixins, serializers, viewsets + +from bats_ai.core.models import CompressedSpectrogram + + +class CompressedSpectrogramSerializer(serializers.ModelSerializer): + class Meta: + model = CompressedSpectrogram + fields = '__all__' + + +class CompressedSpectrogramViewSet( + mixins.ListModelMixin, + viewsets.GenericViewSet, + LargeImageFileDetailMixin, +): + queryset = CompressedSpectrogram.objects.all() + serializer_class = CompressedSpectrogramSerializer + + FILE_FIELD_NAME = 'image_file' diff --git a/bats_ai/core/rest/image.py b/bats_ai/core/rest/image.py deleted file mode 100644 index d749110..0000000 --- a/bats_ai/core/rest/image.py +++ /dev/null @@ -1,42 +0,0 @@ -from django.http import HttpResponseRedirect -from django_filters import rest_framework as filters -from rest_framework import serializers, status -from rest_framework.decorators import action -from rest_framework.pagination import PageNumberPagination -from rest_framework.permissions import IsAuthenticatedOrReadOnly -from rest_framework.response import Response -from rest_framework.viewsets import ReadOnlyModelViewSet - -from bats_ai.core.models import Image -from bats_ai.core.tasks import image_compute_checksum - - -class ImageSerializer(serializers.ModelSerializer): - class Meta: - model = Image - fields = ['id', 'name', 'checksum', 'created', 'owner'] - read_only_fields = ['checksum', 'created'] - - -class ImageViewSet(ReadOnlyModelViewSet): - queryset = Image.objects.all() - - permission_classes = [IsAuthenticatedOrReadOnly] - serializer_class = ImageSerializer - - filter_backends = [filters.DjangoFilterBackend] - filterset_fields = ['name', 'checksum'] - - pagination_class = PageNumberPagination - - @action(detail=True, methods=['get']) - def download(self, request, pk=None): - image = self.get_object() - return HttpResponseRedirect(image.blob.url) - - @action(detail=True, methods=['post']) - def compute(self, request, pk=None): - # Ensure that the image exists, so a non-existent pk isn't dispatched - image = self.get_object() - image_compute_checksum.delay(image.pk) - return Response('', status=status.HTTP_202_ACCEPTED) diff --git a/bats_ai/core/rest/spectrogram.py b/bats_ai/core/rest/spectrogram.py new file mode 100644 index 0000000..a9bf136 --- /dev/null +++ b/bats_ai/core/rest/spectrogram.py @@ -0,0 +1,21 @@ +from django_large_image.rest import LargeImageFileDetailMixin +from rest_framework import mixins, serializers, viewsets + +from bats_ai.core.models import Spectrogram + + +class SpectrogramSerializer(serializers.ModelSerializer): + class Meta: + model = Spectrogram + fields = '__all__' + + +class SpectrogramViewSet( + mixins.ListModelMixin, + viewsets.GenericViewSet, + LargeImageFileDetailMixin, +): + queryset = Spectrogram.objects.all() + serializer_class = SpectrogramSerializer + + FILE_FIELD_NAME = 'image_file' diff --git a/bats_ai/core/tasks.py b/bats_ai/core/tasks.py index a2ffdc3..18604e2 100644 --- a/bats_ai/core/tasks.py +++ b/bats_ai/core/tasks.py @@ -1,6 +1,159 @@ +import io +import tempfile + +from PIL import Image from celery import shared_task +import cv2 +from django.core.files import File +import numpy as np +import scipy + +from bats_ai.core.models import Annotations, CompressedSpectrogram, Recording, Spectrogram, colormap + + +def generate_compressed(recording: Recording, spectrogram: Spectrogram): + img = spectrogram.image_np + annotations = Annotations.objects.filter(recording=recording) + + threshold = 0.5 + while True: + canvas = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) + canvas = canvas.astype(np.float32) + + is_light = np.median(canvas) > 128.0 + if is_light: + canvas = 255.0 - canvas + + amplitude = canvas.max(axis=0) + amplitude -= amplitude.min() + amplitude /= amplitude.max() + amplitude[amplitude < threshold] = 0.0 + amplitude[amplitude > 0] = 1.0 + amplitude = amplitude.reshape(1, -1) + + canvas -= canvas.min() + canvas /= canvas.max() + canvas *= 255.0 + canvas *= amplitude + canvas = np.around(canvas).astype(np.uint8) + + width = canvas.shape[1] + for annotation in annotations: + start = annotation.start_time / spectrogram.duration + stop = annotation.end_time / spectrogram.duration + + start = int(np.around(start * width)) + stop = int(np.around(stop * width)) + canvas[:, start : stop + 1] = 255.0 + + mask = canvas.max(axis=0) + mask = scipy.signal.medfilt(mask, 3) + mask[0] = 0 + mask[-1] = 0 + + starts = [] + stops = [] + for index in range(1, len(mask) - 1): + value_pre = mask[index - 1] + value = mask[index] + value_post = mask[index + 1] + if value != 0: + if value_pre == 0: + starts.append(index) + if value_post == 0: + stops.append(index) + assert len(starts) == len(stops) + + starts = [val - 40 for val in starts] # 10 ms buffer + stops = [val + 40 for val in stops] # 10 ms buffer + ranges = list(zip(starts, stops)) + + while True: + found = False + merged = [] + index = 0 + while index < len(ranges) - 1: + start1, stop1 = ranges[index] + start2, stop2 = ranges[index + 1] + + start1 = min(max(start1, 0), len(mask)) + start2 = min(max(start2, 0), len(mask)) + stop1 = min(max(stop1, 0), len(mask)) + stop2 = min(max(stop2, 0), len(mask)) + + if stop1 >= start2: + found = True + merged.append((start1, stop2)) + index += 2 + else: + merged.append((start1, stop1)) + index += 1 + if index == len(ranges) - 1: + merged.append((start2, stop2)) + ranges = merged + if not found: + for index in range(1, len(ranges)): + start1, stop1 = ranges[index - 1] + start2, stop2 = ranges[index] + assert start1 < stop1 + assert start2 < stop2 + assert start1 < start2 + assert stop1 < stop2 + assert stop1 < start2 + break + + segments = [] + starts_ = [] + stops_ = [] + domain = img.shape[1] + widths = [] + total_width = 0 + for start, stop in ranges: + segment = img[:, start:stop] + segments.append(segment) -from bats_ai.core.models import Image, Recording + starts_.append(int(round(spectrogram.duration * (start / domain)))) + stops_.append(int(round(spectrogram.duration * (stop / domain)))) + widths.append(stop - start) + total_width += stop - start + + # buffer = np.zeros((len(img), 20, 3), dtype=img.dtype) + # segments.append(buffer) + # segments = segments[:-1] + + if len(segments) > 0: + break + + threshold -= 0.05 + if threshold < 0: + segments = None + break + + if segments is None: + canvas = img.copy() + else: + canvas = np.hstack(segments) + + canvas = Image.fromarray(canvas, 'RGB') + buf = io.BytesIO() + canvas.save(buf, format='JPEG', quality=80) + buf.seek(0) + + # Use temporary files + with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as temp_file: + temp_file_name = temp_file.name + canvas.save(temp_file_name) + + # Read the temporary file + with open(temp_file_name, 'rb') as f: + temp_file_content = f.read() + + # Wrap the content in BytesIO + buf = io.BytesIO(temp_file_content) + name = f'{spectrogram.pk}_spectrogram_compressed.jpg' + image_file = File(buf, name=name) + + return total_width, image_file, widths, starts_, stops_ @shared_task @@ -13,6 +166,51 @@ def image_compute_checksum(image_id: int): @shared_task def recording_compute_spectrogram(recording_id: int): recording = Recording.objects.get(pk=recording_id) - if not recording.has_spectrogram: - recording.spectrogram # compute by simply referenceing the attribute - assert recording.has_spectrogram + + cmaps = [ + None, # Default (dark) spectrogram + 'light', # Light spectrogram + ] + spectrogram_id = None + for cmap in cmaps: + with colormap(cmap): + spectrogram_id_temp = Spectrogram.generate(recording, cmap) + if cmap is None: + spectrogram_id = spectrogram_id_temp + if spectrogram_id is not None: + generate_compress_spectrogram.delay(recording_id, spectrogram_id) + + +@shared_task +def generate_compress_spectrogram(recording_id: int, spectrogram_id: int): + recording = Recording.objects.get(pk=recording_id) + spectrogram = Spectrogram.objects.get(pk=spectrogram_id) + length, image_file, widths, starts, stops = generate_compressed(recording, spectrogram) + found = CompressedSpectrogram.objects.filter(recording=recording, spectrogram=spectrogram) + if found.exists(): + existing = found.first() + existing.length = length + existing.image_file = image_file + existing.widths = widths + existing.starts = starts + existing.stops = stops + existing.cache_invalidated = False + existing.save() + else: + CompressedSpectrogram.objects.create( + recording=recording, + spectrogram=spectrogram, + image_file=image_file, + length=length, + widths=widths, + starts=starts, + stops=stops, + cache_invalidated=False, + ) + + +@shared_task +def predict(compressed_spectrogram_id: int): + compressed_spectrogram = CompressedSpectrogram.objects.get(pk=compressed_spectrogram_id) + label, score, confs = compressed_spectrogram.predict() + return label, score, confs diff --git a/bats_ai/core/views/recording.py b/bats_ai/core/views/recording.py index e3c5e2d..a56bcbb 100644 --- a/bats_ai/core/views/recording.py +++ b/bats_ai/core/views/recording.py @@ -11,7 +11,14 @@ from ninja.files import UploadedFile from ninja.pagination import RouterPaginated -from bats_ai.core.models import Annotations, Recording, Species, TemporalAnnotations +from bats_ai.core.models import ( + Annotations, + CompressedSpectrogram, + Recording, + Species, + TemporalAnnotations, + colormap, +) from bats_ai.core.tasks import recording_compute_spectrogram from bats_ai.core.views.species import SpeciesSchema from bats_ai.core.views.temporal_annotations import ( @@ -268,11 +275,13 @@ def get_spectrogram(request: HttpRequest, id: int): except Recording.DoesNotExist: return {'error': 'Recording not found'} - spectrogram = recording.spectrogram + with colormap(None): + spectrogram = recording.spectrogram spectro_data = { - 'base64_spectrogram': spectrogram.base64, + 'url': spectrogram.image_url, 'spectroInfo': { + 'spectroId': spectrogram.pk, 'width': spectrogram.width, 'height': spectrogram.height, 'start_time': 0, @@ -327,25 +336,30 @@ def get_spectrogram(request: HttpRequest, id: int): def get_spectrogram_compressed(request: HttpRequest, id: int): try: recording = Recording.objects.get(pk=id) - except Recording.DoesNotExist: - return {'error': 'Recording not found'} + compressed_spectrogram = CompressedSpectrogram.objects.filter(recording=id).first() + except compressed_spectrogram.DoesNotExist: + return {'error': 'Compressed Spectrogram'} + except recording.DoesNotExist: + return {'error': 'Recording does not exist'} - spectrogram = recording.spectrogram - compressed, starts, ends, widths, total_width = spectrogram.compressed + with colormap(): + label, score, confs = compressed_spectrogram.predict() + print(label, score, confs) spectro_data = { - 'base64_spectrogram': compressed, + 'url': compressed_spectrogram.image_url, 'spectroInfo': { - 'width': spectrogram.width, + 'spectroId': compressed_spectrogram.pk, + 'width': compressed_spectrogram.spectrogram.width, 'start_time': 0, - 'end_time': spectrogram.duration, - 'height': spectrogram.height, - 'start_times': starts, - 'end_times': ends, - 'low_freq': spectrogram.frequency_min, - 'high_freq': spectrogram.frequency_max, - 'compressedWidth': total_width, - 'widths': widths, + 'end_time': compressed_spectrogram.spectrogram.duration, + 'height': compressed_spectrogram.spectrogram.height, + 'low_freq': compressed_spectrogram.spectrogram.frequency_min, + 'high_freq': compressed_spectrogram.spectrogram.frequency_max, + 'start_times': compressed_spectrogram.starts, + 'end_times': compressed_spectrogram.stops, + 'widths': compressed_spectrogram.widths, + 'compressedWidth': compressed_spectrogram.length, }, } diff --git a/bats_ai/settings.py b/bats_ai/settings.py index f64c13e..ef209a0 100644 --- a/bats_ai/settings.py +++ b/bats_ai/settings.py @@ -35,6 +35,7 @@ def mutate_configuration(configuration: ComposedConfiguration) -> None: # Install additional apps configuration.INSTALLED_APPS += [ 'django.contrib.gis', + 'django_large_image', ] configuration.MIDDLEWARE = [ diff --git a/bats_ai/urls.py b/bats_ai/urls.py index 4801c07..5a684a8 100644 --- a/bats_ai/urls.py +++ b/bats_ai/urls.py @@ -3,15 +3,12 @@ from django.urls import include, path from drf_yasg import openapi from drf_yasg.views import get_schema_view -from rest_framework import permissions, routers +from rest_framework import permissions -from bats_ai.core.rest import ImageViewSet +from bats_ai.core.rest import rest from .api import api -router = routers.SimpleRouter() - - # Some more specific Api Requests # OpenAPI generation schema_view = get_schema_view( @@ -19,17 +16,17 @@ public=True, permission_classes=(permissions.AllowAny,), ) -router.register(r'images', ImageViewSet) urlpatterns = [ path('accounts/', include('allauth.urls')), path('oauth/', include('oauth2_provider.urls')), path('admin/', admin.site.urls), path('api/v1/s3-upload/', include('s3_file_field.urls')), - path('api/v1/', include(router.urls)), + path('api/v1/dynamic/', include(rest.urls)), + path('api/v1/', api.urls), path('api/docs/redoc/', schema_view.with_ui('redoc'), name='docs-redoc'), path('api/docs/swagger/', schema_view.with_ui('swagger'), name='docs-swagger'), - path('api/v1/', api.urls), + path('', include('django_large_image.urls')), ] if settings.DEBUG: diff --git a/client/src/api/api.ts b/client/src/api/api.ts index 8754ee8..fea6f67 100644 --- a/client/src/api/api.ts +++ b/client/src/api/api.ts @@ -108,8 +108,7 @@ export interface UserInfo { id: number; } export interface Spectrogram { - 'base64_spectrogram': string; - url?: string; + url: string; filename?: string; annotations?: SpectrogramAnnotation[]; temporal?: SpectrogramTemporalAnnotation[]; diff --git a/client/src/components/SpectrogramViewer.vue b/client/src/components/SpectrogramViewer.vue index df98b16..9b2f729 100644 --- a/client/src/components/SpectrogramViewer.vue +++ b/client/src/components/SpectrogramViewer.vue @@ -21,12 +21,12 @@ export default defineComponent({ }, props: { image: { - type: Object as PropType, - required: true, + type: Object as PropType, + default: () => undefined, }, spectroInfo: { - type: Object as PropType, - default: () => undefined, + type: Object as PropType, + required: true, }, recordingId: { type: String as PropType, @@ -48,6 +48,7 @@ export default defineComponent({ const scaledWidth = ref(0); const scaledHeight = ref(0); const imageCursorRef: Ref = ref(); + const tileURL = props.spectroInfo.spectroId ? `${window.location.protocol}//${window.location.hostname}:${window.location.port}/api/v1/dynamic/spectrograms/${props.spectroInfo.spectroId}/tiles/{z}/{x}/{y}.png/` : ""; const setCursor = (newCursor: string) => { cursor.value = newCursor; }; @@ -131,32 +132,50 @@ export default defineComponent({ } }; watch([containerRef], () => { - const { naturalWidth, naturalHeight } = props.image; - scaledWidth.value = naturalWidth; - scaledHeight.value = naturalHeight; + scaledWidth.value = props.spectroInfo?.width; + scaledHeight.value = props.spectroInfo?.height; + if (props.image) { + const { naturalWidth, naturalHeight } = props.image; + scaledWidth.value = naturalWidth; + scaledHeight.value = naturalHeight; + + } if (containerRef.value) { if (!geoJS.getGeoViewer().value) { - geoJS.initializeViewer(containerRef.value, naturalWidth, naturalHeight); + geoJS.initializeViewer(containerRef.value, scaledWidth.value, scaledHeight.value, false, props.image ? 'quad' : 'tile', tileURL); geoJS.getGeoViewer().value.geoOn(geo.event.mousemove, mouseMoveEvent); } } - geoJS.drawImage(props.image, naturalWidth, naturalHeight); + if (props.image) { + geoJS.drawImage(props.image, scaledWidth.value, scaledHeight.value); + } else { + const scaledTileWidth = (scaledWidth.value / props.spectroInfo?.width) * 256; + const scaledTileHeight = (scaledHeight.value / props.spectroInfo?.height) * 256; + geoJS.updateMapSize(tileURL, scaledWidth.value, scaledHeight.value, scaledTileWidth, scaledTileHeight); + } initialized.value = true; emit("geoViewerRef", geoJS.getGeoViewer()); }); watch(() => props.spectroInfo, () => { - const { naturalWidth, naturalHeight } = props.image; - scaledWidth.value = naturalWidth; - scaledHeight.value = naturalHeight; - geoJS.resetMapDimensions(naturalWidth, naturalHeight); + + scaledHeight.value = props.spectroInfo?.height; + if (props.image) { + const { naturalWidth, naturalHeight } = props.image; + scaledWidth.value = naturalWidth; + scaledHeight.value = naturalHeight; + + } + geoJS.resetMapDimensions(scaledWidth.value, scaledHeight.value); geoJS.getGeoViewer().value.bounds({ left: 0, top: 0, - bottom: naturalHeight, - right: naturalWidth, + bottom: scaledHeight.value, + right: scaledWidth.value, }); - geoJS.drawImage(props.image, naturalWidth, naturalHeight); + if (props.image) { + geoJS.drawImage(props.image, scaledWidth.value, scaledHeight.value); + } }); const updateAnnotation = async (annotation: SpectrogramAnnotation | SpectrogramTemporalAnnotation) => { @@ -207,20 +226,41 @@ export default defineComponent({ ); const wheelEvent = (event: WheelEvent) => { + let baseWidth = 0; + let baseHeight = 0; + if (props.image) { const { naturalWidth, naturalHeight } = props.image; + baseWidth = naturalWidth; + baseHeight = naturalHeight; + } else if (props.spectroInfo) { + baseWidth = props.spectroInfo.width; + baseHeight = props.spectroInfo.height; + } if (event.ctrlKey) { scaledWidth.value = scaledWidth.value + event.deltaY * -4; - if (scaledWidth.value < naturalWidth) { - scaledWidth.value = naturalWidth; + if (scaledWidth.value < baseWidth) { + scaledWidth.value = baseWidth; } + if (props.image) { geoJS.drawImage(props.image, scaledWidth.value, scaledHeight.value, false); + } else if (tileURL) { + const scaledTileWidth = (scaledWidth.value / baseWidth) * 256; + const scaledTileHeight = (scaledHeight.value / baseHeight) * 256; + geoJS.updateMapSize(tileURL, scaledWidth.value, scaledHeight.value, scaledTileWidth, scaledTileHeight); + } } else if (event.shiftKey) { scaledHeight.value = scaledHeight.value + event.deltaY * -0.25; - if (scaledHeight.value < naturalHeight) { - scaledHeight.value = naturalHeight; + if (scaledHeight.value < baseHeight) { + scaledHeight.value = baseHeight; } + if (props.image) { geoJS.drawImage(props.image, scaledWidth.value, scaledHeight.value, false); + } else { + const scaledTileWidth = (scaledWidth.value / baseWidth) * 256; + const scaledTileHeight = (scaledHeight.value / baseHeight) * 256; + geoJS.updateMapSize(tileURL, scaledWidth.value, scaledHeight.value, scaledTileWidth, scaledTileHeight); + } } const xScale = props.spectroInfo?.compressedWidth ? scaledWidth.value / props.spectroInfo.compressedWidth: scaledWidth.value / (props.spectroInfo?.width || 1) ; const yScale = scaledHeight.value / (props.spectroInfo?.height || 1) ; diff --git a/client/src/components/ThumbnailViewer.vue b/client/src/components/ThumbnailViewer.vue index 3ba31f5..14389bb 100644 --- a/client/src/components/ThumbnailViewer.vue +++ b/client/src/components/ThumbnailViewer.vue @@ -149,7 +149,9 @@ export default defineComponent({ const diff = coords.y - end.y; // How much space to we have to multiply the size of the image yScale.value = (clientHeight.value *0.5) / diff; - geoJS.drawImage(props.image, naturalWidth, naturalHeight*yScale.value); + if (props.image) { + geoJS.drawImage(props.image, naturalWidth, naturalHeight*yScale.value); + } initialized.value = true; nextTick(() => createPolyLayer()); }); @@ -160,14 +162,16 @@ export default defineComponent({ clientHeight.value = containerRef.value.clientHeight; } if (containerRef.value && ! geoJS.getGeoViewer().value) { - geoJS.initializeViewer(containerRef.value, naturalWidth, naturalHeight, true); + geoJS.initializeViewer(containerRef.value, naturalWidth, naturalHeight, true); } const coords = geoJS.getGeoViewer().value.camera().worldToDisplay({x: 0, y:0}); const end = geoJS.getGeoViewer().value.camera().worldToDisplay({x: 0, y:naturalHeight}); const diff = coords.y - end.y; // How much space to we have to multiply the size of the image yScale.value = (clientHeight.value *0.5) / diff; - geoJS.drawImage(props.image, naturalWidth, naturalHeight*yScale.value); + if (props.image) { + geoJS.drawImage(props.image, naturalWidth, naturalHeight*yScale.value); + } initialized.value = true; nextTick(() => createPolyLayer()); diff --git a/client/src/components/geoJS/geoJSUtils.ts b/client/src/components/geoJS/geoJSUtils.ts index c799cba..0e5ff5d 100644 --- a/client/src/components/geoJS/geoJSUtils.ts +++ b/client/src/components/geoJS/geoJSUtils.ts @@ -8,6 +8,9 @@ const useGeoJS = () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any let quadFeature: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let osmLayer: any; + const thumbnail = ref(false); let originalBounds = { @@ -17,6 +20,8 @@ const useGeoJS = () => { right: 1, }; + let originalDimensions = { width: 0, height: 0 }; + const getGeoViewer = () => { return geoViewer; }; @@ -25,11 +30,14 @@ const useGeoJS = () => { sourceContainer: HTMLElement, width: number, height: number, - thumbanilVal = false + thumbnailVal = false, + mapType: 'quad' | 'tile' = 'quad', + tileUrl = "" ) => { - thumbnail.value = thumbanilVal; + thumbnail.value = thumbnailVal; container.value = sourceContainer; - const params = geo.util.pixelCoordinateParams(container.value, width, height); + originalDimensions = {width, height }; + const params = geo.util.pixelCoordinateParams(container.value, width, height, mapType === 'tile' ? 256 : width, mapType === 'tile' ? 256 : height); if (!container.value) { return; } @@ -91,16 +99,44 @@ const useGeoJS = () => { right: width, }); + if (mapType === 'quad') { const quadFeatureLayer = geoViewer.value.createLayer("feature", { features: ["quad"], autoshareRenderer: false, renderer: "canvas", }); quadFeature = quadFeatureLayer.createFeature("quad"); + } else if ( mapType === 'tile') { + const params = geo.util.pixelCoordinateParams( + container.value, width, height, 256, 256); + params.layer.useCredentials = true; + params.layer.autoshareRenderer = false; + params.attributes = null; + params.layer.maxLevel = 18; + params.layer.minLevel = 0; + params.layer.url = tileUrl; + + osmLayer = geoViewer.value.createLayer('osm', params.layer); + resetMapDimensions(width, height); + } + }; + + const updateMapSize = (url ='', width =0, height = 0, tileWidth=256, tileHeight=256, resetCam=true) => { + const params = geo.util.pixelCoordinateParams( + container.value, width, height, tileWidth, tileHeight); + params.layer.url = url; + const tempLayer = osmLayer; + osmLayer = geoViewer.value.createLayer('osm', params.layer); + geoViewer.value.deleteLayer(tempLayer); + if (resetCam) { + resetMapDimensions(width, height, 0.3, resetCam); + } }; - const drawImage = (image: HTMLImageElement, width = image.width, height = image.height, resetCam=true) => { - if (quadFeature) { + const drawImage = (image: HTMLImageElement | string, width = 0, height = 0, resetCam=true) => { + let tilewidth = width; + let tileheight = height; + if (quadFeature && typeof (image) === 'object') { quadFeature .data([ { @@ -110,11 +146,23 @@ const useGeoJS = () => { }, ]) .draw(); - } + } if (resetCam) { - resetMapDimensions(width, height, 0.3,resetCam); + resetMapDimensions(width, height, 0.3, resetCam); } else { - const params = geo.util.pixelCoordinateParams(container.value, width, height, width, height); + const params = geo.util.pixelCoordinateParams(container.value, width, height, tilewidth, tileheight); + if (osmLayer && typeof (image) === 'string') { + osmLayer.url(image); + tilewidth = 256; + tileheight = 256; + + osmLayer._options.maxLevel = params.layer.maxLevel; + osmLayer._options.tileWidth = params.layer.tileWidth; + osmLayer._options.tileHeight = params.layer.tileHeight; + osmLayer._options.tilesAtZoom = params.layer.tilesAtZoom; + osmLayer._options.tilesMaxBounds = params.layer.tilesMaxBounds; + + } const margin = 0.3; const { right, bottom } = params.map.maxBounds; originalBounds = params.map.maxBounds; @@ -129,16 +177,21 @@ const useGeoJS = () => { }; const resetZoom = () => { - const { width: mapWidth } = geoViewer.value.camera().viewport; + const { width: mapWidth, } = geoViewer.value.camera().viewport; const bounds = !thumbnail.value ? { - left: -125, // Making sure the legend is on the screen - top: 0, - right: mapWidth, + left: 0, // Making sure the legend is on the screen + top: -(originalBounds.bottom - originalDimensions.height) / 2.0, + right: mapWidth*2, bottom: originalBounds.bottom, } - : originalBounds; + : { + left: 0, + top: 0, + right: originalDimensions.width, + bottom: originalDimensions.height, + }; const zoomAndCenter = geoViewer.value.zoomAndCenterFromBounds(bounds, 0); geoViewer.value.zoom(zoomAndCenter.zoom); geoViewer.value.center(zoomAndCenter.center); @@ -154,18 +207,18 @@ const useGeoJS = () => { }); const params = geo.util.pixelCoordinateParams(container.value, width, height, width, height); const { right, bottom } = params.map.maxBounds; - originalBounds = params.map.maxBounds; geoViewer.value.maxBounds({ left: 0 - right * margin, top: 0 - bottom * margin, right: right * (1 + margin), bottom: bottom * (1 + margin), }); + originalBounds = geoViewer.value.maxBounds(); geoViewer.value.zoomRange({ // do not set a min limit so that bounds clamping determines min min: -Infinity, // 4x zoom max - max: 4, + max: 20, }); geoViewer.value.clampBoundsX(true); geoViewer.value.clampBoundsY(true); @@ -181,12 +234,14 @@ const useGeoJS = () => { drawImage, resetMapDimensions, resetZoom, + updateMapSize, }; }; import { SpectrogramAnnotation, SpectrogramTemporalAnnotation } from "../../api/api"; export interface SpectroInfo { + spectroId: number; width: number; height: number; start_time: number; diff --git a/client/src/views/Spectrogram.vue b/client/src/views/Spectrogram.vue index 2147323..88a3710 100644 --- a/client/src/views/Spectrogram.vue +++ b/client/src/views/Spectrogram.vue @@ -84,17 +84,25 @@ export default defineComponent({ }; const loadData = async () => { + loadedImage.value = false; const response = compressed.value ? await getSpectrogramCompressed(props.id) : await getSpectrogram(props.id); - image.value.src = `data:image/png;base64,${response.data["base64_spectrogram"]}`; + if (response.data['url']) { + image.value.src = response.data['url']; + } else { + // TODO Error Out if there is no URL + console.error('No URL found for the spectrogram'); + } + image.value.onload = () => { + loadedImage.value = true; + }; spectroInfo.value = response.data["spectroInfo"]; annotations.value = response.data["annotations"]?.sort((a, b) => a.start_time - b.start_time) || []; temporalAnnotations.value = response.data["temporal"]?.sort((a, b) => a.start_time - b.start_time) || []; if (response.data.currentUser) { currentUser.value = response.data.currentUser; } - loadedImage.value = true; const speciesResponse = await getSpecies(); speciesList.value = speciesResponse.data; if (response.data.otherUsers && spectroInfo.value) { @@ -397,7 +405,7 @@ export default defineComponent({ =4.1, <4.2', 'django-allauth', 'django-configurations[database,email]', 'django-extensions', - 'django-filter', + 'django-large-image', 'django-oauth-toolkit', 'djangorestframework', 'drf-yasg', @@ -53,10 +54,21 @@ 'django-s3-file-field[boto3]<1', 'gunicorn', 'flower', + # Spectrogram Generation 'librosa', 'matplotlib', + 'mercantile', 'numpy', + # 'onnxruntime-gpu', + 'onnx', + 'onnxruntime', 'opencv-python-headless', + 'tqdm', + # large image + 'django-large-image>=0.10.0', + 'large-image[rasterio,pil]>=1.22', + 'rio-cogeo', + # guano metadata 'guano', ], extras_require={