From 65c6615a585c16b0386cb795d7bf51e99e68c8bd Mon Sep 17 00:00:00 2001 From: turnerm Date: Wed, 9 May 2018 17:38:41 +0000 Subject: [PATCH 01/14] Changed default noise model to avoid 0 division --- banzai/stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/banzai/stages.py b/banzai/stages.py index 0335a2e30..c5e463b0b 100755 --- a/banzai/stages.py +++ b/banzai/stages.py @@ -226,4 +226,4 @@ def apply_master_calibration(self, images, master_calibration_image, logging_tag @abc.abstractmethod def noise_model(self, image): - return np.zeros(image.data.size) + return np.ones(image.data.size) From adab78705db2b167d8529cb7968fb22e5251cfa6 Mon Sep 17 00:00:00 2001 From: turnerm Date: Fri, 11 May 2018 15:38:05 +0000 Subject: [PATCH 02/14] Modified handling of missing calibration master --- banzai/stages.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/banzai/stages.py b/banzai/stages.py index c5e463b0b..a0779353f 100755 --- a/banzai/stages.py +++ b/banzai/stages.py @@ -160,10 +160,11 @@ def do_stage(self, images): if master_calibration_filename is None: self.on_missing_master_calibration(logging_tags) - - master_calibration_image = Image(self.pipeline_context, - filename=master_calibration_filename) - return self.apply_master_calibration(images, master_calibration_image, logging_tags) + return images + else: + master_calibration_image = Image(self.pipeline_context, + filename=master_calibration_filename) + return self.apply_master_calibration(images, master_calibration_image, logging_tags) @abc.abstractmethod def apply_master_calibration(self, images, master_calibration_image, logging_tags): @@ -182,15 +183,7 @@ class CalibrationComparer(ApplyCalibration): def __init__(self, pipeline_context): super(ApplyCalibration, self).__init__(pipeline_context) - def on_missing_master_calibration(self, logging_tags): - msg = 'No master {caltype} frame exists. Assuming these images are ok.' - self.logger.warning(msg.format(caltype=self.calibration_type), logging_tags) - def apply_master_calibration(self, images, master_calibration_image, logging_tags): - # Short circuit - if master_calibration_image.data is None: - return images - images_to_reject = [] for image in images: From 069ed0f6814ce6ded4591c5a84386216860d58af Mon Sep 17 00:00:00 2001 From: turnerm Date: Tue, 15 May 2018 16:04:15 +0000 Subject: [PATCH 03/14] Made function to get bias level of bias --- banzai/bias.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/banzai/bias.py b/banzai/bias.py index 4b4968c94..e609fa147 100755 --- a/banzai/bias.py +++ b/banzai/bias.py @@ -39,8 +39,7 @@ def make_master_calibration_frame(self, images, image_config, logging_tags): master_bias_filename = self.get_calibration_filename(image_config) logs.add_tag(logging_tags, 'master_bias', os.path.basename(master_bias_filename)) for i, image in enumerate(images): - bias_level_array[i] = stats.sigma_clipped_mean(image.data, 3.5, mask=image.bpm) - + bias_level_array[i] = _get_bias_level(image) logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename)) logs.add_tag(logging_tags, 'BIASLVL', float(bias_level_array[i])) self.logger.debug('Calculating bias level', extra=logging_tags) @@ -216,3 +215,8 @@ def _subtract_overscan_2d(image): image.header['OVERSCAN'] = (overscan_level, 'Overscan value that was subtracted') image.data -= overscan_level return overscan_level + + +def _get_bias_level(image): + return stats.sigma_clipped_mean(image.data, 3.5, mask=image.bpm) + From 54aa2b95bac9200b364dc9dd287cf7459d5722b4 Mon Sep 17 00:00:00 2001 From: turnerm Date: Tue, 15 May 2018 16:07:10 +0000 Subject: [PATCH 04/14] Fixed calibration comparison log message --- banzai/stages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/banzai/stages.py b/banzai/stages.py index a0779353f..c34c87fef 100755 --- a/banzai/stages.py +++ b/banzai/stages.py @@ -208,7 +208,8 @@ def apply_master_calibration(self, images, master_calibration_image, logging_tag images_to_reject.append(image) qc_results['rejected'] = True logs.add_tag(logging_tags, 'REJECTED', True) - self.logger.error('Rejecting flat image because it deviates too much from the previous master', + msg = 'Rejecting {0} image because it deviates too much from the previous master' + self.logger.error(msg.format(self.calibration_type), extra=logging_tags) self.save_qc_results(qc_results, image) From 4b60764cb69093cac015e1d0e5915757e95770b8 Mon Sep 17 00:00:00 2001 From: turnerm Date: Tue, 15 May 2018 16:13:18 +0000 Subject: [PATCH 05/14] Changed FLAT to SKYFLAT --- banzai/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/banzai/main.py b/banzai/main.py index a25c6f130..3bcac6cd7 100755 --- a/banzai/main.py +++ b/banzai/main.py @@ -450,7 +450,7 @@ def on_message(self, body, message): # Increment the number of tries for this file dbs.increment_preview_try_number(path, db_address=self.pipeline_context.db_address) - run(stages_to_do, self.pipeline_context, image_types=['EXPOSE', 'STANDARD', 'BIAS', 'DARK', 'FLAT']) + run(stages_to_do, self.pipeline_context, image_types=['EXPOSE', 'STANDARD', 'BIAS', 'DARK', 'SKYFLAT']) dbs.set_preview_file_as_processed(path, db_address=self.pipeline_context.db_address) except Exception as e: From 5dac0911f96fa4b3de74b5a1c90505ec9519f7de Mon Sep 17 00:00:00 2001 From: turnerm Date: Tue, 15 May 2018 17:06:39 +0000 Subject: [PATCH 06/14] Reverted to master --- banzai/bias.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/banzai/bias.py b/banzai/bias.py index e609fa147..4b4968c94 100755 --- a/banzai/bias.py +++ b/banzai/bias.py @@ -39,7 +39,8 @@ def make_master_calibration_frame(self, images, image_config, logging_tags): master_bias_filename = self.get_calibration_filename(image_config) logs.add_tag(logging_tags, 'master_bias', os.path.basename(master_bias_filename)) for i, image in enumerate(images): - bias_level_array[i] = _get_bias_level(image) + bias_level_array[i] = stats.sigma_clipped_mean(image.data, 3.5, mask=image.bpm) + logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename)) logs.add_tag(logging_tags, 'BIASLVL', float(bias_level_array[i])) self.logger.debug('Calculating bias level', extra=logging_tags) @@ -215,8 +216,3 @@ def _subtract_overscan_2d(image): image.header['OVERSCAN'] = (overscan_level, 'Overscan value that was subtracted') image.data -= overscan_level return overscan_level - - -def _get_bias_level(image): - return stats.sigma_clipped_mean(image.data, 3.5, mask=image.bpm) - From 41fe0df48e93a581f36022150fc6cf125c3a3c57 Mon Sep 17 00:00:00 2001 From: turnerm Date: Tue, 15 May 2018 17:46:23 +0000 Subject: [PATCH 07/14] Reverted some of CalibrationComparer --- banzai/stages.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/banzai/stages.py b/banzai/stages.py index c34c87fef..5c82943c5 100755 --- a/banzai/stages.py +++ b/banzai/stages.py @@ -160,11 +160,11 @@ def do_stage(self, images): if master_calibration_filename is None: self.on_missing_master_calibration(logging_tags) - return images else: master_calibration_image = Image(self.pipeline_context, - filename=master_calibration_filename) - return self.apply_master_calibration(images, master_calibration_image, logging_tags) + filename=master_calibration_filename) + images = self.apply_master_calibration(images, master_calibration_image, logging_tags) + return images @abc.abstractmethod def apply_master_calibration(self, images, master_calibration_image, logging_tags): @@ -183,7 +183,15 @@ class CalibrationComparer(ApplyCalibration): def __init__(self, pipeline_context): super(ApplyCalibration, self).__init__(pipeline_context) + def on_missing_master_calibration(self, logging_tags): + msg = 'No master {caltype} frame exists. Assuming these images are ok.' + self.logger.warning(msg.format(caltype=self.calibration_type), logging_tags) + def apply_master_calibration(self, images, master_calibration_image, logging_tags): + # Short circuit + if master_calibration_image.data is None: + return images + images_to_reject = [] for image in images: @@ -208,8 +216,7 @@ def apply_master_calibration(self, images, master_calibration_image, logging_tag images_to_reject.append(image) qc_results['rejected'] = True logs.add_tag(logging_tags, 'REJECTED', True) - msg = 'Rejecting {0} image because it deviates too much from the previous master' - self.logger.error(msg.format(self.calibration_type), + self.logger.error('Rejecting flat image because it deviates too much from the previous master', extra=logging_tags) self.save_qc_results(qc_results, image) @@ -220,4 +227,4 @@ def apply_master_calibration(self, images, master_calibration_image, logging_tag @abc.abstractmethod def noise_model(self, image): - return np.ones(image.data.size) + return np.zeros(image.data.size) From 7555d192b436ed0785e59021bf2070665a9f29fa Mon Sep 17 00:00:00 2001 From: turnerm Date: Tue, 15 May 2018 20:51:05 +0000 Subject: [PATCH 08/14] Remove level subtraction bias subtraction class --- banzai/bias.py | 1 - 1 file changed, 1 deletion(-) diff --git a/banzai/bias.py b/banzai/bias.py index 4b4968c94..a124e793f 100755 --- a/banzai/bias.py +++ b/banzai/bias.py @@ -96,7 +96,6 @@ def apply_master_calibration(self, images, master_calibration_image, logging_tag for image in images: logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename)) - image.subtract(master_bias_level) image.subtract(master_bias_data) image.bpm |= master_calibration_image.bpm From 85154dc4d6a308db866abd27680928dfd4aec6e6 Mon Sep 17 00:00:00 2001 From: turnerm Date: Tue, 15 May 2018 20:52:57 +0000 Subject: [PATCH 09/14] Fix behaviour when site or inst are None --- banzai/images.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/banzai/images.py b/banzai/images.py index faa8c3353..9d8bf7c08 100644 --- a/banzai/images.py +++ b/banzai/images.py @@ -42,6 +42,11 @@ def __init__(self, pipeline_context, filename=None, data=None, header=None, self.site = header.get('SITEID') self.instrument = header.get('INSTRUME') + if self.site is not None and self.instrument is not None: + self.telescope_id = dbs.get_telescope_id(self.site, self.instrument, + db_address=pipeline_context.db_address) + else: + self.telescope_id = None self.epoch = str(header.get('DAY-OBS')) self.nx = header.get('NAXIS1') self.ny = header.get('NAXIS2') @@ -55,8 +60,6 @@ def __init__(self, pipeline_context, filename=None, data=None, header=None, self.ccdsum = header.get('CCDSUM') self.filter = header.get('FILTER') - self.telescope_id = dbs.get_telescope_id(self.site, self.instrument, - db_address=pipeline_context.db_address) self.obstype = header.get('OBSTYPE') self.exptime = float(header.get('EXPTIME')) From f31ca39f87d7d0b07ab1bf39cae8ce5072de6912 Mon Sep 17 00:00:00 2001 From: turnerm Date: Tue, 15 May 2018 20:53:44 +0000 Subject: [PATCH 10/14] Bias master level subtraction is distinct stage --- banzai/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/banzai/main.py b/banzai/main.py index 3bcac6cd7..628dde6cf 100755 --- a/banzai/main.py +++ b/banzai/main.py @@ -37,6 +37,7 @@ mosaic.MosaicCreator, bpm.BPMUpdater, trim.Trimmer, + bias.BiasMasterLevelSubtractor, bias.BiasSubtractor, dark.DarkSubtractor, flats.FlatDivider, @@ -77,8 +78,8 @@ def get_stages_todo(last_stage=None, extra_stages=None): def get_preview_stages_todo(image_suffix): if image_suffix == 'b00.fits': - stages = get_stages_todo(last_stage=trim.Trimmer, - extra_stages=[bias.BiasMasterLevelSubtractor, bias.BiasComparer]) + stages = get_stages_todo(last_stage=bias.BiasMasterLevelSubtractor, + extra_stages=[bias.BiasComparer]) elif image_suffix == 'd00.fits': stages = get_stages_todo(last_stage=bias.BiasSubtractor, extra_stages=[dark.DarkNormalizer, dark.DarkComparer]) From f3ce257d3c67f8d8336ca33f92f771b113bf1d69 Mon Sep 17 00:00:00 2001 From: turnerm Date: Tue, 15 May 2018 20:58:40 +0000 Subject: [PATCH 11/14] Reverted missing calibration error handling --- banzai/stages.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/banzai/stages.py b/banzai/stages.py index 5c82943c5..c5e463b0b 100755 --- a/banzai/stages.py +++ b/banzai/stages.py @@ -160,11 +160,10 @@ def do_stage(self, images): if master_calibration_filename is None: self.on_missing_master_calibration(logging_tags) - else: - master_calibration_image = Image(self.pipeline_context, + + master_calibration_image = Image(self.pipeline_context, filename=master_calibration_filename) - images = self.apply_master_calibration(images, master_calibration_image, logging_tags) - return images + return self.apply_master_calibration(images, master_calibration_image, logging_tags) @abc.abstractmethod def apply_master_calibration(self, images, master_calibration_image, logging_tags): @@ -227,4 +226,4 @@ def apply_master_calibration(self, images, master_calibration_image, logging_tag @abc.abstractmethod def noise_model(self, image): - return np.zeros(image.data.size) + return np.ones(image.data.size) From 70f7c3b1c61e70e1b7737b19705cc0e519d51f1f Mon Sep 17 00:00:00 2001 From: turnerm Date: Tue, 15 May 2018 21:26:23 +0000 Subject: [PATCH 12/14] Update to not consider level subtraction --- banzai/tests/test_bias_subtractor.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/banzai/tests/test_bias_subtractor.py b/banzai/tests/test_bias_subtractor.py index 78a77a20a..672c128b3 100644 --- a/banzai/tests/test_bias_subtractor.py +++ b/banzai/tests/test_bias_subtractor.py @@ -96,13 +96,12 @@ def test_raises_exception_if_no_master_calibration(mock_cal, mock_images): @mock.patch('banzai.stages.ApplyCalibration.get_calibration_filename') def test_bias_subtraction_is_reasonable(mock_cal, mock_image): mock_cal.return_value = 'test.fits' - input_bias = 1000.0 input_readnoise = 9.0 input_level = 2000.0 nx = 101 ny = 103 - fake_master_bias = FakeBiasImage(bias_level=input_bias) + fake_master_bias = FakeBiasImage(bias_level=input_level) fake_master_bias.data = np.random.normal(0.0, input_readnoise, size=(ny, nx)) mock_image.return_value = fake_master_bias @@ -112,5 +111,4 @@ def test_bias_subtraction_is_reasonable(mock_cal, mock_image): images = subtractor.do_stage(images) for image in images: - assert np.abs(image.header['BIASLVL'][0] - input_bias) < 1.0 - assert np.abs(np.mean(image.data) - input_level + input_bias) < 1.0 + np.testing.assert_allclose(input_level-fake_master_bias.data, image.data, 1e-5) From b8c54f0568958bbe2c2a360e3bda77308dc7b3b8 Mon Sep 17 00:00:00 2001 From: turnerm Date: Tue, 15 May 2018 22:03:48 +0000 Subject: [PATCH 13/14] Move bias stage split to another branch --- banzai/bias.py | 1 + banzai/main.py | 5 ++--- banzai/tests/test_bias_subtractor.py | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/banzai/bias.py b/banzai/bias.py index a124e793f..4b4968c94 100755 --- a/banzai/bias.py +++ b/banzai/bias.py @@ -96,6 +96,7 @@ def apply_master_calibration(self, images, master_calibration_image, logging_tag for image in images: logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename)) + image.subtract(master_bias_level) image.subtract(master_bias_data) image.bpm |= master_calibration_image.bpm diff --git a/banzai/main.py b/banzai/main.py index 628dde6cf..3bcac6cd7 100755 --- a/banzai/main.py +++ b/banzai/main.py @@ -37,7 +37,6 @@ mosaic.MosaicCreator, bpm.BPMUpdater, trim.Trimmer, - bias.BiasMasterLevelSubtractor, bias.BiasSubtractor, dark.DarkSubtractor, flats.FlatDivider, @@ -78,8 +77,8 @@ def get_stages_todo(last_stage=None, extra_stages=None): def get_preview_stages_todo(image_suffix): if image_suffix == 'b00.fits': - stages = get_stages_todo(last_stage=bias.BiasMasterLevelSubtractor, - extra_stages=[bias.BiasComparer]) + stages = get_stages_todo(last_stage=trim.Trimmer, + extra_stages=[bias.BiasMasterLevelSubtractor, bias.BiasComparer]) elif image_suffix == 'd00.fits': stages = get_stages_todo(last_stage=bias.BiasSubtractor, extra_stages=[dark.DarkNormalizer, dark.DarkComparer]) diff --git a/banzai/tests/test_bias_subtractor.py b/banzai/tests/test_bias_subtractor.py index 672c128b3..78a77a20a 100644 --- a/banzai/tests/test_bias_subtractor.py +++ b/banzai/tests/test_bias_subtractor.py @@ -96,12 +96,13 @@ def test_raises_exception_if_no_master_calibration(mock_cal, mock_images): @mock.patch('banzai.stages.ApplyCalibration.get_calibration_filename') def test_bias_subtraction_is_reasonable(mock_cal, mock_image): mock_cal.return_value = 'test.fits' + input_bias = 1000.0 input_readnoise = 9.0 input_level = 2000.0 nx = 101 ny = 103 - fake_master_bias = FakeBiasImage(bias_level=input_level) + fake_master_bias = FakeBiasImage(bias_level=input_bias) fake_master_bias.data = np.random.normal(0.0, input_readnoise, size=(ny, nx)) mock_image.return_value = fake_master_bias @@ -111,4 +112,5 @@ def test_bias_subtraction_is_reasonable(mock_cal, mock_image): images = subtractor.do_stage(images) for image in images: - np.testing.assert_allclose(input_level-fake_master_bias.data, image.data, 1e-5) + assert np.abs(image.header['BIASLVL'][0] - input_bias) < 1.0 + assert np.abs(np.mean(image.data) - input_level + input_bias) < 1.0 From 55e24088651ac5ab28b7e6b7846faf9750656a2c Mon Sep 17 00:00:00 2001 From: turnerm Date: Fri, 18 May 2018 15:53:43 +0000 Subject: [PATCH 14/14] Added change included with fix/calibratoin_checker merge --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index 9811672ca..3aac53cc0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ ------------------ - Fixed a bug that would stop preview frames from being retried if they failed even once. - Hotfix to remove double division by exposure time when creating master darks +- Fixed bug that prevented calibration comparison being run on skyflats 0.9.3 (2018-05-10) ------------------