diff --git a/specreduce/background.py b/specreduce/background.py index 47f3524..db8ced1 100644 --- a/specreduce/background.py +++ b/specreduce/background.py @@ -140,6 +140,10 @@ def __post_init__(self): raise ValueError("background regions overlapped") if np.any(np.sum(bkg_wimage, axis=self.crossdisp_axis) == 0): raise ValueError("background window does not remain in bounds across entire dispersion axis") # noqa + if np.all(img.mask[bkg_wimage > 0]): + raise ValueError("Image is fully masked within background window determined by `width`.") + + # check if image contained within background window is fully-nonfinite and raise an error if so if self.statistic == 'median': # make it clear in the expose image that partial pixels are fully-weighted @@ -333,6 +337,7 @@ def sub_image(self, image=None): `~specutils.Spectrum1D` object with same shape as ``image``. """ image = self._parse_image(image) + print(image.data) # a compare_wcs argument is needed for Spectrum1D.subtract() in order to # avoid a TypeError from SpectralCoord when image's spectral axis is in diff --git a/specreduce/core.py b/specreduce/core.py index ec47700..87b801d 100644 --- a/specreduce/core.py +++ b/specreduce/core.py @@ -1,5 +1,6 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst +from copy import deepcopy import inspect from dataclasses import dataclass @@ -140,6 +141,9 @@ def _mask_and_nonfinite_data_handling(self, image, mask): if mask_treatment == 'zero-fill': + # make a copy of the input image since we will be modifying it + image = deepcopy(image) + # if mask_treatment is 'zero_fill', set masked values to zero in # image data and drop image.mask. note that this is done after # _combine_mask_with_nonfinite_from_data, so non-finite values in diff --git a/specreduce/tests/test_background.py b/specreduce/tests/test_background.py index 9b34c84..0f9834d 100644 --- a/specreduce/tests/test_background.py +++ b/specreduce/tests/test_background.py @@ -1,3 +1,4 @@ +from astropy.nddata import NDData import astropy.units as u import numpy as np import pytest @@ -113,7 +114,11 @@ def test_warnings_errors(mk_test_spec_no_spectral_axis): def test_trace_inputs(mk_test_img_raw): - # Tests for the input argument 'traces' to `Background`. + """ + Tests for the input argument 'traces' to `Background`. This should accept + a list of or a single Trace object, or a list of or a single (positive) + number to define a FlatTrace. + """ image = mk_test_img_raw @@ -140,60 +145,102 @@ def test_trace_inputs(mk_test_img_raw): class TestMasksBackground(): - def mk_img(self, nrows=4, ncols=5, nan_slices=None, add_noise=True): + """ + Various test functions to test how masked and non-finite data is handled + in `Background. + """ - # make a image to test masking in Background - # image is n columns, each col_i = i + noise + def mk_img(self, nrows=4, ncols=5, nan_slices=None): + """ + Make a simpleimage to test masking in Background. + Optionally add NaNs to data. Returned array is in u.DN. + """ img = np.tile((np.arange(1., ncols+1)), (nrows, 1)) - noise = 0 - if add_noise: - np.random.seed(7) - sigma_noise = 0.2 - noise = np.random.normal(scale=sigma_noise, size=(nrows, ncols)) - img += noise - if nan_slices: # add nans in data for s in nan_slices: img[s] = np.nan return img * u.DN + def test_fully_masked(self): + """ + Test that the appropriate error is raised by `Background` when image + is fully masked/NaN. + """ + + with pytest.raises(ValueError, match='Image is fully masked.'): + # fully NaN image + img = np.zeros((4, 5)) * np.nan + Background(img, traces=FlatTrace(self.mk_img(), 2)) + + with pytest.raises(ValueError, match='Image is fully masked.'): + # fully masked image (should be equivilant) + img = NDData(np.ones((4, 5)), mask=np.ones((4, 5))) + Background(img, traces=FlatTrace(self.mk_img(), 2)) + + # Now test that an image that isn't fully masked, but is fully masked + # within the window determined by `width`, produces the correct result + msg = 'Image is fully masked within background window determined by `width`.' + with pytest.raises(ValueError, match=msg): + img = self.mk_img(nrows=12, ncols=12, nan_slices=[np.s_[3:10, :]]) + Background(img, traces=FlatTrace(img, 6), width=7) + + @pytest.mark.filterwarnings("ignore:background window extends beyond image boundaries") @pytest.mark.parametrize("method,expected", [("filter", np.array([1., 2., 3., 4., 5., 6., 7., - 8., 9., 10., 11., 12.]))]) + 8., 9., 10., 11., 12.])), + ("omit", np.array([0., 2., 3., 0., 5., 6., + 7., 0., 9., 10., 11., 12.])), + ("zero-fill", np.array([ 0.58333333, 2., 3., + 2.33333333, 5., 6., 7., + 7.33333333, 9., 10., 11., + 12.]))]) def test_mask_treatment_bkg_img_spectrum(self, method, expected): - - """This test function tests creating a Background object, and computing - `bkg_image` and `bkg_spectrum` when there is masked data. The test - image has NaNs, which are added to the masked array. This test is - parameterized over all currently implemented mask handling methods - (filter, omit, and zero-fill) to test that all three work as - intended.""" + """ + This test function tests `Backgroud.bkg_image` and + `Background.bkg_spectrum` when there is masked data. It also tests + background subtracting the image, and returning the spectrum of the + background subtracted image. This test is parameterized over all + currently implemented mask handling methods (filter, omit, and + zero-fill) to test that all three work as intended. The window size is + set to use the entire image array, so warning about background window + is ignored.""" img_size = 12 # square 12 x 12 image - # make image, set some value to nan so they are included in mask - image = self.mk_img(nrows=img_size, ncols=img_size, + # make image, set some value to nan, which will be masked in the function + image1 = self.mk_img(nrows=img_size, ncols=img_size, nan_slices=[np.s_[5:10, 0], np.s_[7:12, 3], - np.s_[2, 7]], add_noise=False) - - # construct a flat trace in center of image - trace = FlatTrace(image, img_size/2) - - # create 'Background' with `mask_treatment` set - background = Background(image, mask_treatment=method, - traces=trace) - - # test background image - bk_img = background.bkg_image() - # change this and following assertions to assert_quantity_allclose once - # issue #213 is fixed - np.testing.assert_allclose(bk_img.flux.value, - np.tile(expected, (img_size, 1))) - - # test background spectrum - bk_spec = background.bkg_spectrum() - np.testing.assert_allclose(bk_spec.flux.value, expected * img_size) - + np.s_[2, 7]]) + + # also make an image that doesn't have nonf data values, but has + # masked values at the same locations, to make sure they give the same + # results + mask = ~np.isfinite(image1) + dat = self.mk_img(nrows=img_size, ncols=img_size) + image2 = NDData(dat, mask=mask) + + for image in [image1, image2]: + + # construct a flat trace in center of image + trace = FlatTrace(image, img_size/2) + + # create 'Background' object with `mask_treatment` set + # 'width' should be > size of image to use all pix (but warning will + # be raised, which we ignore.) + background = Background(image, mask_treatment=method, + traces=trace, width=img_size+1) + + # test background image matches 'expected' + bk_img = background.bkg_image() + # change this and following assertions to assert_quantity_allclose once + # issue #213 is fixed + np.testing.assert_allclose(bk_img.flux.value, + np.tile(expected, (img_size, 1))) + + # test background spectrum matches 'expected' times the number of rows + # since this is a sum + bk_spec = background.bkg_spectrum() + np.testing.assert_allclose(bk_spec.flux.value, expected * img_size) diff --git a/specreduce/tests/test_tracing.py b/specreduce/tests/test_tracing.py index 3e79ba1..4beb509 100644 --- a/specreduce/tests/test_tracing.py +++ b/specreduce/tests/test_tracing.py @@ -155,7 +155,10 @@ class TestMasksTracing(): def mk_img(self, nrows=200, ncols=160, nan_slices=None, add_noise=True): - # make a gaussian image for testing + """ + Makes a gaussian image for testing, with optional added gaussian + nosie and optional data values set to NaN. + """ sigma_pix = 4 col_model = models.Gaussian1D(amplitude=1, mean=nrows/2, @@ -176,10 +179,12 @@ def mk_img(self, nrows=200, ncols=160, nan_slices=None, add_noise=True): return img * u.DN def test_window_fit_trace(self): - - """This test function will test that masked values are treated correctly in + """ + This test function will test that masked values are treated correctly in FitTrace, and produce the correct results and warning messages based on - `peak_method`.""" + `peak_method`. + """ + img = self.mk_img() # create same-shaped variations of image with invalid values @@ -201,10 +206,15 @@ def test_window_fit_trace(self): FitTrace(img_all_nans) @pytest.mark.filterwarnings("ignore:All pixels in bins") - def test_fit_trace_all_nan_cols(self): - - # make sure that the actual trace that is fit is correct when - # all-masked bin peaks are set to NaN + @pytest.mark.parametrize("mask_treatment", ['filter', 'omit']) + def test_fit_trace_all_nan_cols(self, mask_treatment): + """ + Create a test image that has some fully-masked columns, and test that + when the final fit to all bin peaks is done for the trace, that these + fully-masked columns are set to NaN and filtered during the fit. This + should happen for mask_treatment = 'filter' and 'omit' (for 'zero-fill', + all NaN columns become all-zero columns). + """ img = self.mk_img(nrows=10, ncols=11) img[:, 7] = np.nan @@ -215,14 +225,16 @@ def test_fit_trace_all_nan_cols(self): truth = [1.6346154, 2.2371795, 2.8397436, 3.4423077, 4.0448718, 4.6474359, 5.25, 5.8525641, 6.4551282, 7.0576923, 7.6602564] - max_trace = FitTrace(img, peak_method='max') + max_trace = FitTrace(img, peak_method='max', + mask_treatment=mask_treatment) np.testing.assert_allclose(truth, max_trace.trace) # peak_method = 'gaussian' truth = [1.947455, 2.383634, 2.8198131, 3.2559921, 3.6921712, 4.1283502, 4.5645293, 5.0007083, 5.4368874, 5.8730665, 6.3092455] - max_trace = FitTrace(img, peak_method='gaussian') + max_trace = FitTrace(img, peak_method='gaussian', + mask_treatment=mask_treatment) np.testing.assert_allclose(truth, max_trace.trace) # peak_method = 'centroid' @@ -230,11 +242,18 @@ def test_fit_trace_all_nan_cols(self): 3.7828113, 4.0329969, 4.2831824, 4.533368, 4.7835536, 5.0337391] max_trace = FitTrace(img, peak_method='centroid') - np.testing.assert_allclose(truth, max_trace.trace) + np.testing.assert_allclose(truth, max_trace.trace, + mask_treatment=mask_treatment) def test_warn_msg_fit_trace_all_nan_cols(self): - + """ + Test that the correct warning is raised when fully masked columns + are encountered in FitTrace. These columns will be set to NaN and filtered + from the final all-bin fit. This should happen for + mask_treatment='filter' and 'omit' (for 'zero-fill', all NaN columns + become all-zero columns). + """ img = self.mk_img() # test that warning (dependent on choice of `peak_method`) is raised when a @@ -276,35 +295,41 @@ def test_warn_msg_fit_trace_all_nan_cols(self): np.nan, 4.27108332, 4.27108332, 4.27108332, 1.19673467, 4.27108332])]) def test_mask_treatment_filter(self, peak_method, expected): - # Test for mask_treatment=filter for FitTrace. - # With this masking option, masked and nonfinite data should be filtered - # when determining bin/column peak. Fully masked bins should be omitted - # from the final all-bin-peak fit for the Trace. - - # In this test, noise is not added to simple gaussian image to make - # sanity check of results easier to understand. - - # make an image with some nonfinite values in some columns, which will be - # added to a mask within FitTrace - imgg = self.mk_img(nan_slices=[np.s_[4:8, 1:2], np.s_[2:7, 4:5], + """ + Test for mask_treatment=filter for FitTrace. + With this masking option, masked and nonfinite data should be filtered + when determining bin/column peak. Fully masked bins should be omitted + from the final all-bin-peak fit for the Trace. Parametrized over different + `peak_method` options. + """ + + # Make an image with some nonfinite values. + image1 = self.mk_img(nan_slices=[np.s_[4:8, 1:2], np.s_[2:7, 4:5], np.s_[:, 6:7], np.s_[3:9, 10:11]], nrows=10, ncols=12, add_noise = False) - # run FitTrace, with the testing-only flag _save_bin_peaks_testing set - # to True to return the bin peak values before fitting the trace - trace = FitTrace(imgg, peak_method=peak_method, - _save_bin_peaks_testing=True) - x_bins, y_bins = trace._bin_peaks_testing - np.testing.assert_allclose(y_bins, expected) - - # check that final fit to all bins, accouting for fully-masked bins, - # matches the trace - fitter = fitting.LevMarLSQFitter() - mask = np.isfinite(y_bins) - all_bin_fit = fitter(trace.trace_model, x_bins[mask], y_bins[mask]) - all_bin_fit = all_bin_fit((np.arange(imgg.shape[1]))) - - np.testing.assert_allclose(trace.trace, all_bin_fit) + # Also make an image that doesn't have nonf data values, but has masked + # values at the same locations, to make sure they give the same results. + mask = ~np.isfinite(image1) + dat = self.mk_img(nrows=10, ncols=12, add_noise=False) + image2 = NDData(dat, mask=mask) + + for imgg in [image1, image2]: + # run FitTrace, with the testing-only flag _save_bin_peaks_testing set + # to True to return the bin peak values before fitting the trace + trace = FitTrace(imgg, peak_method=peak_method, + _save_bin_peaks_testing=True) + x_bins, y_bins = trace._bin_peaks_testing + np.testing.assert_allclose(y_bins, expected) + + # check that final fit to all bins, accouting for fully-masked bins, + # matches the trace + fitter = fitting.LevMarLSQFitter() + mask = np.isfinite(y_bins) + all_bin_fit = fitter(trace.trace_model, x_bins[mask], y_bins[mask]) + all_bin_fit = all_bin_fit((np.arange(12))) + + np.testing.assert_allclose(trace.trace, all_bin_fit) @pytest.mark.filterwarnings("ignore:All pixels in bins") @pytest.mark.parametrize("peak_method,expected", @@ -317,34 +342,41 @@ def test_mask_treatment_filter(self, peak_method, expected): 9., 4.27108332, 4.27108332, 4.27108332, 1.19673467, 4.27108332])]) def test_mask_treatment_zero_fill(self, peak_method, expected): - # Test for mask_treatment=`zero_fill` for FitTrace. - # Masked and nonfinite data are replaced with zero in the data array, - # and the input mask is then dropped. - - # In this test, noise is not added to simple gaussian image to make - # sanity check of results easier to understand. - - # make an image with some nonfinite values in some columns, which will be - # added to a mask within FitTrace - imgg = self.mk_img(nan_slices=[np.s_[4:8, 1:2], np.s_[2:7, 4:5], + """ + Test for mask_treatment=`zero_fill` for FitTrace. + Masked and nonfinite data are replaced with zero in the data array, + and the input mask is then dropped. Parametrized over different + `peak_method` options. + """ + + # Make an image with some nonfinite values. + image1 = self.mk_img(nan_slices=[np.s_[4:8, 1:2], np.s_[2:7, 4:5], np.s_[:, 6:7], np.s_[3:9, 10:11]], nrows=10, ncols=12, add_noise = False) - # run FitTrace, with the testing-only flag _save_bin_peaks_testing set - # to True to return the bin peak values before fitting the trace - trace = FitTrace(imgg, peak_method=peak_method, mask_treatment='zero-fill', - _save_bin_peaks_testing=True) - x_bins, y_bins = trace._bin_peaks_testing - np.testing.assert_allclose(y_bins, expected) - - # check that final fit to all bins, accouting for fully-masked bins, - # matches the trace - fitter = fitting.LevMarLSQFitter() - mask = np.isfinite(y_bins) - all_bin_fit = fitter(trace.trace_model, x_bins[mask], y_bins[mask]) - all_bin_fit = all_bin_fit((np.arange(imgg.shape[1]))) - - np.testing.assert_allclose(trace.trace, all_bin_fit) + # Also make an image that doesn't have nonf data values, but has masked + # values at the same locations, to make sure they give the same results. + mask = ~np.isfinite(image1) + dat = self.mk_img(nrows=10, ncols=12, add_noise=False) + image2 = NDData(dat, mask=mask) + + for imgg in [image1, image2]: + # run FitTrace, with the testing-only flag _save_bin_peaks_testing set + # to True to return the bin peak values before fitting the trace + trace = FitTrace(imgg, peak_method=peak_method, + mask_treatment='zero-fill', + _save_bin_peaks_testing=True) + x_bins, y_bins = trace._bin_peaks_testing + np.testing.assert_allclose(y_bins, expected) + + # check that final fit to all bins, accouting for fully-masked bins, + # matches the trace + fitter = fitting.LevMarLSQFitter() + mask = np.isfinite(y_bins) + all_bin_fit = fitter(trace.trace_model, x_bins[mask], y_bins[mask]) + all_bin_fit = all_bin_fit((np.arange(12))) + + np.testing.assert_allclose(trace.trace, all_bin_fit) @pytest.mark.filterwarnings("ignore:All pixels in bins") @pytest.mark.parametrize("peak_method,expected", @@ -357,31 +389,38 @@ def test_mask_treatment_zero_fill(self, peak_method, expected): np.nan, 4.27108332, 4.27108332, 4.27108332, np.nan, 4.27108332])]) def test_mask_treatment_omit(self, peak_method, expected): - # Test for mask_treatment=`omit` for FitTrace. - # Columns (assuming horizontal trace) with any masked data values will - # be fully masked. - - # In this test, noise is not added to simple gaussian image to make - # sanity check of results easier to understand. - - # make an image with some nonfinite values in some columns, which will be - # added to a mask within FitTrace - imgg = self.mk_img(nan_slices=[np.s_[4:8, 1:2], np.s_[2:7, 4:5], + """ + Test for mask_treatment=`omit` for FitTrace. Columns (assuming + disp_axis==1) with any masked data values will be fully masked and + therefore not contribute to the bin peaks. Parametrized over different + `peak_method` options.""" + + # Make an image with some nonfinite values. + image1 = self.mk_img(nan_slices=[np.s_[4:8, 1:2], np.s_[2:7, 4:5], np.s_[:, 6:7], np.s_[3:9, 10:11]], nrows=10, ncols=12, add_noise = False) - # run FitTrace, with the testing-only flag _save_bin_peaks_testing set - # to True to return the bin peak values before fitting the trace - trace = FitTrace(imgg, peak_method=peak_method, mask_treatment='omit', - _save_bin_peaks_testing=True) - x_bins, y_bins = trace._bin_peaks_testing - np.testing.assert_allclose(y_bins, expected) - - # check that final fit to all bins, accouting for fully-masked bins, - # matches the trace - fitter = fitting.LevMarLSQFitter() - mask = np.isfinite(y_bins) - all_bin_fit = fitter(trace.trace_model, x_bins[mask], y_bins[mask]) - all_bin_fit = all_bin_fit((np.arange(imgg.shape[1]))) - - np.testing.assert_allclose(trace.trace, all_bin_fit) + # Also make an image that doesn't have nonf data values, but has masked + # values at the same locations, to make sure they give the same results. + mask = ~np.isfinite(image1) + dat = self.mk_img(nrows=10, ncols=12, add_noise=False) + image2 = NDData(dat, mask=mask) + + for imgg in [image1, image2]: + + # run FitTrace, with the testing-only flag _save_bin_peaks_testing set + # to True to return the bin peak values before fitting the trace + trace = FitTrace(imgg, peak_method=peak_method, + mask_treatment='omit', + _save_bin_peaks_testing=True) + x_bins, y_bins = trace._bin_peaks_testing + np.testing.assert_allclose(y_bins, expected) + + # check that final fit to all bins, accouting for fully-masked bins, + # matches the trace + fitter = fitting.LevMarLSQFitter() + mask = np.isfinite(y_bins) + all_bin_fit = fitter(trace.trace_model, x_bins[mask], y_bins[mask]) + all_bin_fit = all_bin_fit((np.arange(12))) + + np.testing.assert_allclose(trace.trace, all_bin_fit)