Skip to content

Commit

Permalink
Axis Scaling (#61)
Browse files Browse the repository at this point in the history 
* experiment scaling

* X/Y scaling

* compressed X axis support

* fix compressed editing
  • Loading branch information
@BryonLewis
BryonLewis authored Feb 25, 2024
1 parent c58455d commit 390910a
Show file tree
Hide file tree
Showing 15 changed files with 376 additions and 95 deletions.
4 changes: 3 additions & 1 deletion bats_ai/core/models/spectrogram.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,13 +239,15 @@ def compressed(self):
stops_ = []
domain = img.shape[1]
widths = []
total_width = 0
for start, stop in ranges:
segment = img[:, start:stop]
segments.append(segment)

starts_.append(int(round(self.duration * (start / domain))))
stops_.append(int(round(self.duration * (stop / domain))))
widths.append(stop - start)
total_width += stop - start

# buffer = np.zeros((len(img), 20, 3), dtype=img.dtype)
# segments.append(buffer)
Expand Down Expand Up @@ -273,7 +275,7 @@ def compressed(self):
buf.seek(0)
img_base64 = base64.b64encode(buf.getvalue()).decode('utf-8')

return img_base64, starts_, stops_, widths
return img_base64, starts_, stops_, widths, total_width

@property
def image_np(self):
Expand Down
12 changes: 11 additions & 1 deletion bats_ai/core/views/recording.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@ def get_spectrogram_compressed(request: HttpRequest, id: int):
return {'error': 'Recording not found'}

spectrogram = recording.spectrogram
compressed, starts, ends, widths = spectrogram.compressed
compressed, starts, ends, widths, total_width = spectrogram.compressed

spectro_data = {
'base64_spectrogram': compressed,
Expand All @@ -287,6 +287,7 @@ def get_spectrogram_compressed(request: HttpRequest, id: int):
'end_times': ends,
'low_freq': spectrogram.frequency_min,
'high_freq': spectrogram.frequency_max,
'compressedWidth': total_width,
'widths': widths,
},
}
Expand Down Expand Up @@ -314,13 +315,22 @@ def get_spectrogram_compressed(request: HttpRequest, id: int):
spectro_data['currentUser'] = request.user.email

annotations_qs = Annotations.objects.filter(recording=recording, owner=request.user)
temporal_annotations_qs = TemporalAnnotations.objects.filter(
recording=recording, owner=request.user
)

# Serialize the annotations using AnnotationSchema
annotations_data = [
AnnotationSchema.from_orm(annotation, owner_email=request.user.email).dict()
for annotation in annotations_qs
]
temporal_annotations_data = [
TemporalAnnotationSchema.from_orm(annotation, owner_email=request.user.email).dict()
for annotation in temporal_annotations_qs
]

spectro_data['annotations'] = annotations_data
spectro_data['temporal'] = temporal_annotations_data
return spectro_data


Expand Down
76 changes: 58 additions & 18 deletions client/src/components/SpectrogramViewer.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,13 @@ export default defineComponent({
"hoverData",
],
setup(props, { emit }) {
const { annotations, temporalAnnotations, selectedId, selectedType, creationType, blackBackground } = useState();
const { annotations, temporalAnnotations, selectedId, selectedType, creationType, blackBackground, scaledVals } = useState();
const containerRef: Ref<HTMLElement | undefined> = ref();
const geoJS = useGeoJS();
const initialized = ref(false);
const cursor = ref("");
const scaledWidth = ref(0);
const scaledHeight = ref(0);
const imageCursorRef: Ref<HTMLElement | undefined> = ref();
const setCursor = (newCursor: string) => {
cursor.value = newCursor;
Expand Down Expand Up @@ -77,20 +79,23 @@ export default defineComponent({
if (!props.spectroInfo) {
return;
}
const adjustedWidth = scaledWidth.value > props.spectroInfo.width ? scaledWidth.value : props.spectroInfo.width;
const adjustedHeight = scaledHeight.value > props.spectroInfo.height ? scaledHeight.value : props.spectroInfo.height;
const freq =
props.spectroInfo.height - y >= 0
? ((props.spectroInfo.height - y) *
(props.spectroInfo.high_freq - props.spectroInfo.low_freq)) /
props.spectroInfo.height /
1000 +
props.spectroInfo.low_freq / 1000
adjustedHeight - y >= 0
? ((adjustedHeight - y) *
(props.spectroInfo.high_freq - props.spectroInfo.low_freq)) /
adjustedHeight /
1000 +
props.spectroInfo.low_freq / 1000
: -1;
if (!props.spectroInfo.end_times && !props.spectroInfo.start_times) {
if (x >= 0 && props.spectroInfo.height - y >= 0) {
if (x >= 0 && adjustedHeight - y >= 0) {
const time =
x *
((props.spectroInfo.end_time - props.spectroInfo.start_time) / props.spectroInfo.width);
((props.spectroInfo.end_time - props.spectroInfo.start_time) / adjustedWidth);
emit("hoverData", { time, freq });
} else {
emit("hoverData", { time: -1, freq: -1 });
Expand All @@ -101,9 +106,9 @@ export default defineComponent({
props.spectroInfo.end_times
) {
// compressed view
if (x >= 0 && props.spectroInfo.height - y >= 0) {
if (x >= 0 && adjustedHeight - y >= 0) {
const timeLength = props.spectroInfo.end_time - props.spectroInfo.start_time;
const timeToPixels = props.spectroInfo.width / timeLength;
const timeToPixels = adjustedWidth / timeLength;
// find X in the range
let offsetAdditive = 0;
for (let i = 0; i < props.spectroInfo.start_times.length; i += 1) {
Expand All @@ -127,6 +132,8 @@ export default defineComponent({
};
watch(containerRef, () => {
const { naturalWidth, naturalHeight } = props.image;
scaledWidth.value = naturalWidth;
scaledHeight.value = naturalHeight;
if (containerRef.value)
geoJS.initializeViewer(containerRef.value, naturalWidth, naturalHeight);
geoJS.drawImage(props.image, naturalWidth, naturalHeight);
Expand Down Expand Up @@ -166,9 +173,9 @@ export default defineComponent({
if (skipNextSelected) {
skipNextSelected = false;
return;
}
const found = selectedType.value === 'pulse' ? annotations.value.find((item) => item.id === selectedId.value): temporalAnnotations.value.find((item) => item.id === selectedId.value);
const found = selectedType.value === 'pulse' ? annotations.value.find((item) => item.id === selectedId.value) : temporalAnnotations.value.find((item) => item.id === selectedId.value);
if (found && props.spectroInfo) {
const center = spectroToCenter(found, props.spectroInfo, selectedType.value);
Expand All @@ -182,6 +189,28 @@ export default defineComponent({
}
);
const wheelEvent = (event: WheelEvent) => {
const { naturalWidth, naturalHeight } = props.image;
if (event.ctrlKey) {
scaledWidth.value = scaledWidth.value + event.deltaY * -4;
if (scaledWidth.value < naturalWidth) {
scaledWidth.value = naturalWidth;
}
geoJS.drawImage(props.image, scaledWidth.value, scaledHeight.value, false);
} else if (event.shiftKey) {
scaledHeight.value = scaledHeight.value + event.deltaY * -0.25;
if (scaledHeight.value < naturalHeight) {
scaledHeight.value = naturalHeight;
}
geoJS.drawImage(props.image, scaledWidth.value, scaledHeight.value, false);
}
const xScale = props.spectroInfo?.compressedWidth ? scaledWidth.value / props.spectroInfo.compressedWidth: scaledWidth.value / (props.spectroInfo?.width || 1) ;
const yScale = scaledHeight.value / (props.spectroInfo?.height || 1) ;
scaledVals.value = {x: xScale, y: yScale};
};
return {
containerRef,
Expand All @@ -194,6 +223,9 @@ export default defineComponent({
cursorHandler,
imageCursorRef,
blackBackground,
wheelEvent,
scaledWidth,
scaledHeight,
};
},
});
Expand All @@ -202,7 +234,8 @@ export default defineComponent({
<template>
<div
class="video-annotator"
:class="{'black-background': blackBackground, 'white-background': !blackBackground}"
:class="{ 'black-background': blackBackground, 'white-background': !blackBackground }"
@wheel="wheelEvent($event)"
>
<div
id="spectro"
Expand All @@ -217,12 +250,14 @@ export default defineComponent({
v-if="initialized"
:geo-viewer-ref="geoViewerRef"
:spectro-info="spectroInfo"
:scaled-width="scaledWidth"
:scaled-height="scaledHeight"
@update:annotation="updateAnnotation($event)"
@create:annotation="createAnnotation($event)"
@set-cursor="setCursor($event)"
/>
<div
ref="imageCursorRef"
ref="imageCursorRef"
class="imageCursor"
>
<v-icon color="white">
Expand All @@ -243,8 +278,10 @@ export default defineComponent({
display: flex;
flex-direction: column;
.geojs-map {
margin: 2px;
&.geojs-map:focus {
outline: none;
}
Expand All @@ -254,6 +291,7 @@ export default defineComponent({
.playback-container {
flex: 1;
}
.loadingSpinnerContainer {
z-index: 20;
margin: 0;
Expand All @@ -263,18 +301,20 @@ export default defineComponent({
-ms-transform: translate(-50%, -50%);
transform: translate(-50%, -50%);
}
.geojs-map.annotation-input {
cursor: inherit;
}
}
.black-background {
background-color: black;
}
background-color: black;
}
.white-background {
background-color: white;
background-color: white;
}
.imageCursor {
z-index: 9999; //So it will be above the annotator layers
position: fixed;
Expand Down
Loading

0 comments on commit 390910a

Please sign in to comment.