Add FIlterSet and InstrumentFilterSet models

calibrations/admin.py

@@ -1,7 +1,9 @@
 from django.contrib import admin
-from .models import Filter, Instrument, InstrumentFilter
+from .models import Filter, FilterSet, Instrument, InstrumentFilterSet
 
 # Register your models here
 admin.site.register(Filter)
+admin.site.register(FilterSet)
 admin.site.register(Instrument)
-admin.site.register(InstrumentFilter)
+#admin.site.register(InstrumentFilter)
+admin.site.register(InstrumentFilterSet)

calibrations/cadences/photometric_standards_cadence.py

@@ -13,7 +13,7 @@
 from tom_targets.models import Target
 
 from configdb.configdb_connections import ConfigDBInterface
-from calibrations.models import Filter, Instrument, InstrumentFilter
+from calibrations.models import Filter, FilterSet, Instrument, InstrumentFilterSet
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
@@ -41,10 +41,13 @@ def update_observation_payload(self, observation_payload):
     def update_observation_filters(self, observation_payload):
         logger.info(msg='Updating observation_payload filters')
         instrument = Instrument.objects.get(code=self.dynamic_cadence.cadence_parameters['instrument_code'])
+        #logger.info(msg=f'instrument : {instrument}')
         filter_dates = []
+        logger.info(msg=f'instrumentfilter_set : {instrument.instrumentfilter_set.all()}')
         for inst_filter in instrument.instrumentfilter_set.all():
+            logger.info(msg=f'inst_filter : {inst_filter}')
             filter_dates.append([inst_filter, inst_filter.get_last_calibration_age(self.dynamic_cadence.observation_group)])  # TODO: explore an annotation instead
-
+        
         # float('inf') returns infinity, thus guaranteeing that filters with calibration age of None will be considered
         # as the oldest calibrations
         filter_dates.sort(key=lambda filters: filters[1] if filters[1] is not None else float('inf'), reverse=True)
@@ -58,6 +61,35 @@ def update_observation_filters(self, observation_payload):
 
         return observation_payload
 
+    def update_observation_filterset(self, observation_payload):
+        logger.info(msg=f'Updating observation_payload filter set')
+        instrument = Instrument.objects.get(code=self.dynamic_cadence.cadence_parameters['instrument_code'])
+        #logger.info(msg=f'instrument : {instrument}')
+
+        filterset_dates = [] 
+        logger.info(msg=f'instrumentfilterset_set : {instrument.instrumentfilterset_set.all()}')
+        for filter_set in instrument.instrumentfilterset_set.all(): # iterate through all filtersets on this instrument
+            logger.info(msg=f'filter_set : {filter_set}')
+            filterset_dates.append((filter_set, filter_set.get_last_instrumentfilterset_age(self.dynamic_cadence.observation_group)))  
+            # Above: each element in the filterset_dates list is a tuple with element 1 = filterset, element 2 = age determined by get_last_instrumentfilterset_age
+
+        filterset_dates.sort(key=lambda filterset: filterset[1] if filterset[1] is not None else float('inf'), reverse=True) # sorts by age
+        # Above: float('inf') returns infinity, thus guaranteeing that filters with calibration age of None will be considered the oldest calibrations
+
+        oldest_instrumentfilterset, age = filterset_dates[0]  # select only the oldest filterset
+        #logger.info(f'oldest_instrumentfilterset : {oldest_instrumentfilterset} is {age} days old.')
+
+        for instrument_filter_set in instrument.instrumentfilterset_set.all(): # iterate through all filtersets on this instrument
+
+            for filter in instrument_filter_set.filter_set.filter_combination.all():
+                observation_payload[f'{filter.name}_selected'] = False # De-select each filter in each instrument_filter_set
+
+        for filter in oldest_instrumentfilterset.filter_set.filter_combination.all(): # iterate through each filter in the oldest filterset
+
+            observation_payload[f'{filter.name}_selected'] = True # Select the filters that belong to this filterset
+
+        return observation_payload
+
     def run(self):
         last_obs = self.dynamic_cadence.observation_group.observation_records.order_by('-created').first()
         target = Target.objects.get(pk=self.dynamic_cadence.cadence_parameters['target_id'])
@@ -191,7 +223,8 @@ def run(self):
             observation_payload = self.advance_window(
                 observation_payload, start_keyword=start_keyword, end_keyword=end_keyword
             )
-            observation_payload = self.update_observation_filters(observation_payload)
+            #observation_payload = self.update_observation_filters(observation_payload)
+            observation_payload = self.update_observation_filterset(observation_payload)
 
         observation_payload = self.update_observation_payload(observation_payload)
 

calibrations/migrations/0004_filterset_instrumentfilterset.py

@@ -0,0 +1,30 @@
+# Generated by Django 4.2.10 on 2024-10-23 22:47
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('calibrations', '0003_filter_instrument_instrumentfilter'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='FilterSet',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('filter_set_code', models.ManyToManyField(to='calibrations.filter')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='InstrumentFilterSet',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('max_age', models.IntegerField(default=5)),
+                ('filter_set', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='calibrations.filterset')),
+                ('instrument', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='calibrations.instrument')),
+            ],
+        ),
+    ]

calibrations/models.py

@@ -43,7 +43,7 @@ class Filter(models.Model):
     )
 
     def __str__(self):
-        return f'{self.name} filter (default: {self.exposure_count} x {self.exposure_time} second exposures)'
+        return f'{self.name} ({self.exposure_count} exp x {self.exposure_time} s)'
 
     # Boilerplate fields
     created = models.DateTimeField(
@@ -55,6 +55,15 @@ def __str__(self):
         help_text='The time which this target was changed in the TOM database.'
     )
 
+class FilterSet(models.Model):
+    filter_combination = models.ManyToManyField(Filter) # map filtersets to filters
+
+    def __str__(self):
+        filters = self.filter_combination.all()
+        description = ' '
+        for filter in filters:
+            description += f'{filter.name} ' # display filters in filter set
+        return description
 
 class Instrument(models.Model):
     site = models.CharField(max_length=3)
@@ -80,12 +89,56 @@ def get_last_calibration_age(self, observation_group=None):
             records = ObservationRecord.objects.all()
         else:
             records = observation_group.observation_records.all()
+
         kwargs = {f'parameters__{self.filter.name}_selected': True,
                   'parameters__instrument': self.instrument.code,
                   'status': 'COMPLETED'}
+
         last_calibration = records.order_by('-created').filter(**kwargs).first()
+
         if last_calibration and last_calibration.scheduled_end:
             return (datetime.now(timezone.utc) - last_calibration.scheduled_end).days
 
     def __str__(self):
         return f'{self.instrument.code} - {self.filter.name}'
+
+class InstrumentFilterSet(models.Model):
+    instrument = models.ForeignKey(Instrument, on_delete=models.CASCADE)
+    filter_set = models.ForeignKey(FilterSet, on_delete=models.CASCADE)
+    max_age = models.IntegerField(default=5)
+
+    def get_last_instrumentfilterset_age(self, observation_group=None):
+        if not observation_group:
+            records = ObservationRecord.objects.all()
+        else:
+            records = observation_group.observation_records.all()
+
+        ages = []
+        filterset = self.filter_set.filter_combination.all()
+        #print('filterset = ', filterset)
+
+        for filter in filterset: # loop through each filter in set
+
+            kwargs = {f'parameters__{filter.name}_selected': True,
+                  'parameters__instrument': self.instrument.code,
+                  'status': 'COMPLETED'}
+
+            last_calibration = records.order_by('-created').filter(**kwargs).first()
+            #print('last_calibration = ', last_calibration)
+            #print('last_calibration.scheduled_end = ', last_calibration.scheduled_end)
+
+            age = 0
+            if last_calibration and last_calibration.scheduled_end:
+                age = (datetime.now(timezone.utc) - last_calibration.scheduled_end).days
+
+            ages.append(age) # create list of filter ages in set
+
+        filterset_age = max(ages) # return only the age of the oldest filter in the set
+
+        return filterset_age
+
+    def __str__(self):
+        ic = self.instrument.code
+        fs = self.filter_set
+
+        return f'{ic} : {fs}'

helm-chart/Chart.yaml

@@ -1,4 +1,4 @@
-apiVersion: v1
+apiVersion: v2
 appVersion: "1.0"
 description: A Helm chart for Kubernetes
 name: calibration-tom

helm-chart/values-prod.yaml

@@ -43,6 +43,23 @@ importinstruments:
       cpu: 1000m
       memory: 1024Mi
 
+runcadencestrategies:
+  resources:
+    requests:
+      cpu: 100m
+      memory: 256Mi
+    limits:
+      cpu: 1000m
+      memory: 1024Mi
+
+updatestatus:
+  resources:
+    requests:
+      cpu: 100m
+      memory: 256Mi
+    limits:
+      cpu: 1000m
+      memory: 1024Mi
 
 lcoServices:
   configdbURL: "http://configdb.lco.gtn"

requirements.txt

@@ -1,7 +1,21 @@
 # TODO: switch to setup.py so test requirements can be separated
 deprecation~=2.1
-#tomtoolkit~=2.14
-tomtoolkit<3
+
+tomtoolkit==2.19.6
+
+Django==4.2.10
+django-bootstrap4==24.1
+django-contrib-comments==2.2.0
+django-crispy-forms==2.1
+django-dramatiq==0.11.6
+django-extensions==3.2.3
+django-filter==23.5
+django-gravatar2==1.4.4
+django-guardian==2.4.0
+django-storages==1.11.1
+djangorestframework~=3.15
+
+
 crispy-bootstrap4
 psycopg2-binary~=2.9
 gunicorn[gevent]==20.0.4