From e2c06e3036c56d36593efa872beb525bf4bc0cae Mon Sep 17 00:00:00 2001 From: Akash Date: Tue, 30 Apr 2024 22:27:41 -0500 Subject: [PATCH 1/7] Don't offset RMS maps and use sigma clipping to select non-variable sources --- vast_post_processing/core.py | 60 +++++++++++-------- vast_post_processing/corrections.py | 91 ++++++++++++++++++++--------- 2 files changed, 98 insertions(+), 53 deletions(-) diff --git a/vast_post_processing/core.py b/vast_post_processing/core.py index 2a13c8a..4b163d7 100644 --- a/vast_post_processing/core.py +++ b/vast_post_processing/core.py @@ -1,7 +1,6 @@ """Core Pipeline Entry Point for VAST Post-Processing. """ - # Imports @@ -154,7 +153,21 @@ def setup_configuration( overwrite: Optional[bool] = None, verbose: Optional[bool] = None, debug: Optional[bool] = None, -) -> tuple[Path, Path, list[str], list[int], u.Quantity, bool, bool, str, str, str, bool, bool, bool]: +) -> tuple[ + Path, + Path, + list[str], + list[int], + u.Quantity, + bool, + bool, + str, + str, + str, + bool, + bool, + bool, +]: """Set up the configuration settings for this run. Parameters @@ -201,8 +214,7 @@ def setup_configuration( Valid configuration settings for this run. """ # Load in default configuration - default_config = yaml.safe_load( - open(DATA_DIRECTORY / "default_config.yaml")) + default_config = yaml.safe_load(open(DATA_DIRECTORY / "default_config.yaml")) # Load in user configuration if passed and valid, otherwise create empty # dict @@ -262,7 +274,7 @@ def get_image_paths( verbose: bool, debug: bool, image_type: str = "IMAGES", - processed_dir_suffix: str = "PROCESSED" + processed_dir_suffix: str = "PROCESSED", ) -> list[Path]: """Get paths to all FITS images for a given Stokes parameter and epoch. @@ -329,8 +341,7 @@ def get_image_paths( # Check for processed data if Stokes V if parameter == "V": # Display progress if requested - logger.info( - "Checking corresponding Stokes I files have been processed.") + logger.info("Checking corresponding Stokes I files have been processed.") # Get list of Stokes I processed image paths as str # NOTE image_type may change in future development @@ -354,7 +365,9 @@ def get_image_paths( # Get expected path of processed corresponding Stokes I image split_str_path_v = str(image_path_v).split("STOKESV_IMAGES") str_path_i = ( - split_str_path_v[0] + f"STOKESI_IMAGES_{processed_dir_suffix}" + split_str_path_v[1].replace("image.v", "image.i") + split_str_path_v[0] + + f"STOKESI_IMAGES_{processed_dir_suffix}" + + split_str_path_v[1].replace("image.v", "image.i") ) # If processed path is not found, terminate run @@ -458,8 +471,8 @@ def crop_image( overwrite: bool, verbose: bool, debug: bool, - file_extension: str = '.processed.fits', - processed_dir_suffix = "PROCESSED", + file_extension: str = ".processed.fits", + processed_dir_suffix="PROCESSED", ) -> tuple[SkyCoord, fits.PrimaryHDU]: """Crop and compress data corresponding to image data. @@ -517,7 +530,7 @@ def crop_image( # Crop image data if file_extension is None or file_extension == "": - file_extension = '.fits' + file_extension = ".fits" out_name = path.name.replace(".fits", file_extension) outfile = fits_output_dir / out_name image_hdu = corrected_fits[i] @@ -568,8 +581,8 @@ def crop_catalogs( overwrite: bool, verbose: bool, debug: bool, - file_extension: str = '.processed.xml', - processed_dir_suffix: str = 'PROCESSED', + file_extension: str = ".processed.xml", + processed_dir_suffix: str = "PROCESSED", ): """Crop field catalogues. @@ -608,10 +621,10 @@ def crop_catalogs( cat_output_dir.mkdir(parents=True, exist_ok=True) # Iterate over each catalogue xml file corresponding to a field - for i, path in enumerate((components_path, )): + for i, path in enumerate((components_path,)): # Path to output file if file_extension is None or file_extension == "": - file_extension = '.xml' + file_extension = ".xml" out_name = path.name.replace(".xml", file_extension) outfile = cat_output_dir / out_name @@ -623,8 +636,7 @@ def crop_catalogs( # This uses the last cropped hdu from the previous for loop # which should be the image file, but doesn't actually matter - cropped_vot = crop.crop_catalogue( - vot, cropped_hdu, field_centre, crop_size) + cropped_vot = crop.crop_catalogue(vot, cropped_hdu, field_centre, crop_size) # Add git hash catutils.add_hash_to_cat(cropped_vot) @@ -648,7 +660,7 @@ def create_mocs( overwrite: bool, verbose: bool, debug: bool, - processed_dir_suffix: str = 'PROCESSED', + processed_dir_suffix: str = "PROCESSED", ): """Create a MOC and STMOC for a given field image. @@ -801,7 +813,7 @@ def run( # Set up logger main_logger = logutils.setup_logger(verbose=verbose, debug=debug) - if stokes != ['I'] and stokes != 'I' and create_moc: + if stokes != ["I"] and stokes != "I" and create_moc: main_logger.warning("Stokes != I, so setting create_moc=False") create_moc = False @@ -816,7 +828,7 @@ def run( epoch=epoch, processed_dir_suffix=directory_suffix, verbose=verbose, - debug=debug + debug=debug, ) # Display paths if requested @@ -832,7 +844,7 @@ def run( field, sbid = misc.get_field_and_sbid(image_path) # Get and verify relevant paths for this file - rms_path, bkg_path, components_path = get_corresponding_paths( + rms_path, bkg_path, components_path = get_corresponding_paths( data_root=data_root, image_path=image_path, stokes=stokes_dir, @@ -850,7 +862,7 @@ def run( overwrite=overwrite, verbose=verbose, debug=debug, - write_output=False + write_output=False, ) # Display corrected files if requested @@ -905,7 +917,7 @@ def run( overwrite=overwrite, verbose=verbose, debug=debug, - processed_dir_suffix=directory_suffix + processed_dir_suffix=directory_suffix, ) # Create MOCs @@ -919,5 +931,5 @@ def run( overwrite=overwrite, verbose=verbose, debug=debug, - processed_dir_suffix=directory_suffix + processed_dir_suffix=directory_suffix, ) diff --git a/vast_post_processing/corrections.py b/vast_post_processing/corrections.py index ba7457a..e735fea 100644 --- a/vast_post_processing/corrections.py +++ b/vast_post_processing/corrections.py @@ -1,7 +1,6 @@ """Apply various corrections to FITS image files. """ - # Imports @@ -22,6 +21,7 @@ from astropy.io.votable.tree import Param, VOTableFile, Table from astropy.wcs import WCS, FITSFixedWarning from astropy.coordinates import SkyCoord, Angle +from astropy.stats import sigma_clip import astropy.units as u from .catalogs import Catalog @@ -58,6 +58,7 @@ def vast_xmatch_qc( flux_limit: float = 0, snr_limit: float = 20, nneighbor: float = 1, + outlier_lim: float = 3, apply_flux_limit: bool = True, select_point_sources: bool = True, crossmatch_output: Optional[str] = None, @@ -155,6 +156,14 @@ def vast_xmatch_qc( mask &= xmatch_qt["flux_peak_err_reference"] > 0 mask &= xmatch_qt["has_siblings"] == 0 mask &= xmatch_qt["has_siblings_reference"] == 0 + + # Also use a mask to try to remove outliers + mask &= ~( + sigma_clip( + data=xmatch_qt["flux_peak_ratio"], sigma=outlier_lim, maxiters=None + ).mask + ) + data = xmatch_qt[mask] logger.info( f"{len(data):.0f} crossmatched sources remaining" @@ -292,7 +301,7 @@ def shift_and_scale_image( else: data_unit = u.mJy - flux_offset = (flux_offset_mJy*u.mJy).to(data_unit) + flux_offset = (flux_offset_mJy * u.mJy).to(data_unit) image_hdu.data = flux_scale * image_hdu.data + flux_offset.value image_hdu.header["FLUXOFF"] = flux_offset.value @@ -393,15 +402,15 @@ def shift_and_scale_catalog( "col_flux_int_err", ) - COMPONENT_FLUX_COLS = ( + COMPONENT_FLUX_RES_COLS = ( "col_rms_fit_gauss", "col_rms_image", ) # Flux-unit columns in the island catalogs only - ISLAND_FLUX_COLS = ( - "col_mean_background", - "col_background_noise", + ISLAND_FLUX_COLS = ("col_mean_background",) + ISLAND_FLUX_ERR_COLS = ("col_background_noise",) + ISLAND_FLUX_RES_COLS = ( "col_max_residual", "col_min_residual", "col_mean_residual", @@ -434,15 +443,24 @@ def shift_and_scale_catalog( # Add position corrections ra_err_corrected = ( ra_err**2 - + ((ra_offset_arcsec_err*u.arcsec) / np.cos(dec_rad)) ** 2 - + (dec_err.to(u.radian).value * (ra_offset_arcsec * u.arcsec) * np.tan(dec_rad) / np.cos(dec_rad)) ** 2 + + ((ra_offset_arcsec_err * u.arcsec) / np.cos(dec_rad)) ** 2 + + ( + dec_err.to(u.radian).value + * (ra_offset_arcsec * u.arcsec) + * np.tan(dec_rad) + / np.cos(dec_rad) + ) + ** 2 ) ** 0.5 ra_err_corrected = ra_err_corrected.to(u.arcsec) logger.debug(f"ra_err_corrected: {ra_err_corrected}") logger.debug(f"ra_err: {ra_err}") logger.debug(f"ra_offset_arcsec: {ra_offset_arcsec}") - dec_err_corrected = ((dec_err.to(u.radian))**2 + ((dec_offset_arcsec_err * u.arcsec).to(u.radian)) ** 2) ** 0.5 + dec_err_corrected = ( + (dec_err.to(u.radian)) ** 2 + + ((dec_offset_arcsec_err * u.arcsec).to(u.radian)) ** 2 + ) ** 0.5 logger.debug(f"dec_err_corrected: {dec_err_corrected}") dec_err_corrected = dec_err_corrected.to(u.arcsec) logger.debug(f"dec_err_corrected: {dec_err_corrected}") @@ -472,20 +490,25 @@ def shift_and_scale_catalog( ) # Correct flux columns - cols = ( - FLUX_COLS + ISLAND_FLUX_COLS if is_island else FLUX_COLS + COMPONENT_FLUX_COLS - ) + # cols = FLUX_COLS + ISLAND_FLUX_COLS if is_island else FLUX_COLS + flux_cols = FLUX_COLS + ISLAND_FLUX_COLS if is_island else FLUX_COLS + rms_cols = FLUX_ERR_COLS + ISLAND_FLUX_ERR_COLS if is_island else FLUX_ERR_COLS - for col in cols: + for col in flux_cols: votable.array[col] = flux_scale * votable.array[col] + flux_offset_mJy - for f in zip(FLUX_COLS, FLUX_ERR_COLS): + for f in zip(flux_cols, rms_cols): votable.array[f[1]] = ( flux_scale_err**2 * votable.array[f[0]] ** 2 + flux_scale**2 * votable.array[f[1]] ** 2 + flux_offset_mJy_err**2 ) ** 0.5 + # Now come to residuals and just scale them, no offset + res_cols = ISLAND_FLUX_RES_COLS if is_island else COMPONENT_FLUX_RES_COLS + for col in res_cols: + votable.array[col] = flux_scale * votable.array[col] + # Add in the corrections to the votable flux_scl_param = Param( votable=votablefile, @@ -659,16 +682,12 @@ def check_for_files(image_path: str, stokes: str = "I"): catalog_filepath = f"{catalog_root}/{catalog_filename}" component_file = Path(catalog_filepath) - + skip = ( - not ( - (rms_path.exists()) - and (bkg_path.exists()) - and (component_file.exists()) - ) + not ((rms_path.exists()) and (bkg_path.exists()) and (component_file.exists())) or skip ) - return skip, (bkg_path, rms_path, component_file ) + return skip, (bkg_path, rms_path, component_file) def correct_field( @@ -682,6 +701,7 @@ def correct_field( flux_limit: float = 0, snr_limit: float = 20, nneighbor: float = 1, + outlier_lim: float = 3, apply_flux_limit: bool = True, select_point_sources: bool = True, write_output: bool = True, @@ -781,6 +801,7 @@ def correct_field( snr_limit=snr_limit, nneighbor=nneighbor, apply_flux_limit=apply_flux_limit, + outlier_lim=outlier_lim, select_point_sources=select_point_sources, crossmatch_output=crossmatch_file, csv_output=csv_file, @@ -824,13 +845,23 @@ def correct_field( if output_path.exists() and not overwrite: logger.warning(f"Will not overwrite existing image: {output_path}.") else: - corrected_hdu = shift_and_scale_image( - path, - flux_scale=flux_corr_mult_value, - flux_offset_mJy=flux_corr_add_value, - ra_offset_arcsec=dra_median_value, - dec_offset_arcsec=ddec_median_value, - ) + # Scaling images is fine, but be sure not to offset RMS image + if path == rms_path: + corrected_hdu = shift_and_scale_image( + path, + flux_scale=flux_corr_mult_value, + flux_offset_mJy=0.0, + ra_offset_arcsec=dra_median_value, + dec_offset_arcsec=ddec_median_value, + ) + else: + corrected_hdu = shift_and_scale_image( + path, + flux_scale=flux_corr_mult_value, + flux_offset_mJy=flux_corr_add_value, + ra_offset_arcsec=dra_median_value, + dec_offset_arcsec=ddec_median_value, + ) if write_output: output_dir.mkdir(parents=True, exist_ok=True) if output_path.exists() and overwrite: @@ -843,7 +874,7 @@ def correct_field( # Do the same for catalog files corrected_catalogs = [] - for path in (component_file, ): + for path in (component_file,): stokes_dir = f"{path.parent.parent.name}_CORRECTED" output_dir = outdir / stokes_dir / epoch_dir output_path = output_dir / path.with_suffix(".corrected.xml").name @@ -892,6 +923,7 @@ def correct_files( flux_limit: float = 0, snr_limit: float = 20, nneighbor: float = 1, + outlier_lim: float = 3, apply_flux_limit: bool = True, select_point_sources: bool = True, write_output: bool = True, @@ -971,6 +1003,7 @@ def correct_files( psf=psf, flux_limit=flux_limit, snr_limit=snr_limit, + outlier_lim=outlier_lim, nneighbor=nneighbor, apply_flux_limit=apply_flux_limit, select_point_sources=select_point_sources, From 3b6b2d2491316a8be3f3ec5a0fcd4caf0e319e94 Mon Sep 17 00:00:00 2001 From: Akash Date: Sat, 11 May 2024 10:44:10 -0500 Subject: [PATCH 2/7] Change the default settings to fix intercept to 0; add better docstrings and log information --- vast_post_processing/corrections.py | 83 ++++++++++++++++++++++++----- vast_post_processing/crossmatch.py | 1 - 2 files changed, 71 insertions(+), 13 deletions(-) diff --git a/vast_post_processing/corrections.py b/vast_post_processing/corrections.py index e735fea..8ca8fd4 100644 --- a/vast_post_processing/corrections.py +++ b/vast_post_processing/corrections.py @@ -53,12 +53,14 @@ def vast_xmatch_qc( psf: Optional[Tuple[float, float]] = None, fix_m: bool = False, fix_b: bool = False, + init_m: float = 1, + init_b: float = 0, positional_unit: u.Unit = u.Unit("arcsec"), flux_unit: u.Unit = u.Unit("mJy"), flux_limit: float = 0, snr_limit: float = 20, nneighbor: float = 1, - outlier_lim: float = 3, + flux_ratio_sigma_clip: float = 5, apply_flux_limit: bool = True, select_point_sources: bool = True, crossmatch_output: Optional[str] = None, @@ -88,6 +90,10 @@ def vast_xmatch_qc( fix_b : bool, optional Flag to fix intercept, by default False. TODO re: linear fit - variable or fixed slope? + init_m : float + Initial gradient parameter passed to the fitting function, default 1.0. + init_b : float + Initial offset parameter passed to the fitting function, default 0.0. positional_unit : u.Unit, optional Output unit of astrometric offset, by default u.Unit("arcsec"). flux_unit : u.Unit, optional @@ -101,6 +107,8 @@ def vast_xmatch_qc( that source, by default 1. apply_flux_limit : bool, optional Flag to apply flux limit, by default True. + flux_ratio_sigma_clip : float, optional + Reject all the points outside this value of standard deviation select_point_sources : bool, optional Flag to select point sources, by default True. crossmatch_output : Optional[str], optional @@ -158,12 +166,27 @@ def vast_xmatch_qc( mask &= xmatch_qt["has_siblings_reference"] == 0 # Also use a mask to try to remove outliers - mask &= ~( - sigma_clip( - data=xmatch_qt["flux_peak_ratio"], sigma=outlier_lim, maxiters=None - ).mask + # Do an interative fitting that reoves all the outilers + # beyond n-sigma standard deviation where n is the flux_ratio_sigma_clip + sigma_clip_mask = sigma_clip( + data=xmatch_qt["flux_peak_ratio"], + sigma=flux_ratio_sigma_clip, + maxiters=None, + ).mask + + sigma_clip_ratio = sigma_clip_mask.sum() / len(xmatch_qt) + + logger.info( + f"{sigma_clip_mask.sum()} sources have been clipped out for variability." ) + if sigma_clip_ratio > 0.5: + logger.warning( + f"{sigma_clip_ratio * 100:.2f}% sources are removed for variability." + ) + + mask &= ~(sigma_clip_mask) + data = xmatch_qt[mask] logger.info( f"{len(data):.0f} crossmatched sources remaining" @@ -188,7 +211,7 @@ def vast_xmatch_qc( # Calculate flux offsets and ratio gradient, offset, gradient_err, offset_err = calculate_flux_offsets( - data, fix_m=fix_m, fix_b=fix_b + data, fix_m=fix_m, fix_b=fix_b, init_m=init_m, init_b=init_b ) ugradient = ufloat(gradient, gradient_err) uoffset = ufloat(offset.to(flux_unit).value, offset_err.to(flux_unit).value) @@ -701,7 +724,11 @@ def correct_field( flux_limit: float = 0, snr_limit: float = 20, nneighbor: float = 1, - outlier_lim: float = 3, + flux_ratio_sigma_clip: float = 5, + fix_m: bool = False, + fix_b: bool = True, + init_m: float = 1, + init_b: float = 0, apply_flux_limit: bool = True, select_point_sources: bool = True, write_output: bool = True, @@ -722,6 +749,16 @@ def correct_field( condon (bool, optional): Flag to replace errros with Condon errors. Defaults to True. psf_ref (list[float], optional): PSF information of the reference catalog. Defaults to None. psf (list[float], optional): PSF information of the input catalog. Defaults to None. + init_m : float + Initial gradient parameter passed to the fitting function, default 1.0. + init_b : float + Initial offset parameter passed to the fitting function, default 0.0. + fix_m : bool + If True, do not allow the gradient to vary during fitting, default False. + fix_b : bool + If True, do not allow the offest to vary during fitting, default False. + flux_ratio_sigma_clip : float, optional + Reject all the points outside this value of standard deviation write_output (bool, optional): Write the corrected image and catalog files or return the corrected hdul and the corrected table?. Defaults to True, which means to write outdir (str, optional): The stem of the output directory to write the files to @@ -795,13 +832,15 @@ def correct_field( condon=condon, psf_reference=psf_reference, psf=psf_image, - fix_m=False, - fix_b=False, + fix_m=fix_m, + fix_b=fix_b, + init_m=init_m, + init_b=init_b, flux_limit=flux_limit, snr_limit=snr_limit, nneighbor=nneighbor, apply_flux_limit=apply_flux_limit, - outlier_lim=outlier_lim, + flux_ratio_sigma_clip=flux_ratio_sigma_clip, select_point_sources=select_point_sources, crossmatch_output=crossmatch_file, csv_output=csv_file, @@ -847,6 +886,8 @@ def correct_field( else: # Scaling images is fine, but be sure not to offset RMS image if path == rms_path: + # For RMS maps, additive offset should not be added since changing + # the zero point of flux density should not change the noise level. corrected_hdu = shift_and_scale_image( path, flux_scale=flux_corr_mult_value, @@ -923,7 +964,11 @@ def correct_files( flux_limit: float = 0, snr_limit: float = 20, nneighbor: float = 1, - outlier_lim: float = 3, + fix_m: bool = False, + fix_b: bool = True, + init_m: float = 1, + init_b: float = 0, + flux_ratio_sigma_clip: float = 5, apply_flux_limit: bool = True, select_point_sources: bool = True, write_output: bool = True, @@ -947,6 +992,16 @@ def correct_files( condon (bool, optional): Flag to replace errros with Condon errors. Defaults to True. psf_ref (list[float], optional): PSF information of the reference catalog. Defaults to None. psf (list[float], optional): PSF information of the input catalog. Defaults to None. + init_m : float + Initial gradient parameter passed to the fitting function, default 1.0. + init_b : float + Initial offset parameter passed to the fitting function, default 0.0. + fix_m : bool + If True, do not allow the gradient to vary during fitting, default False. + fix_b : bool + If True, do not allow the offest to vary during fitting, default False. + flux_ratio_sigma_clip (float, optional): + Reject all the points outside this value of standard deviation write_output (bool, optional): Write the corrected image and catalog files or return the corrected hdul and the corrected table?. Defaults to True, which means to write outdir (str, optional): The stem of the output directory to write the files to @@ -1003,7 +1058,11 @@ def correct_files( psf=psf, flux_limit=flux_limit, snr_limit=snr_limit, - outlier_lim=outlier_lim, + fix_m=fix_m, + fix_b=fix_b, + init_m=init_m, + init_b=init_b, + flux_ratio_sigma_clip=flux_ratio_sigma_clip, nneighbor=nneighbor, apply_flux_limit=apply_flux_limit, select_point_sources=select_point_sources, diff --git a/vast_post_processing/crossmatch.py b/vast_post_processing/crossmatch.py index 5bca8bd..cc81dbf 100644 --- a/vast_post_processing/crossmatch.py +++ b/vast_post_processing/crossmatch.py @@ -1,7 +1,6 @@ """Cross-match sources in VAST image observations. """ - # Imports From 6463626e91d125ca781b1f7a67aad3a3e012616f Mon Sep 17 00:00:00 2001 From: Dougal Dobie Date: Mon, 13 May 2024 15:59:28 +1000 Subject: [PATCH 3/7] Fixed sigma clipping issue --- vast_post_processing/corrections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vast_post_processing/corrections.py b/vast_post_processing/corrections.py index 8ca8fd4..78764f6 100644 --- a/vast_post_processing/corrections.py +++ b/vast_post_processing/corrections.py @@ -169,7 +169,7 @@ def vast_xmatch_qc( # Do an interative fitting that reoves all the outilers # beyond n-sigma standard deviation where n is the flux_ratio_sigma_clip sigma_clip_mask = sigma_clip( - data=xmatch_qt["flux_peak_ratio"], + data=np.asarray(xmatch_qt["flux_peak_ratio"]), sigma=flux_ratio_sigma_clip, maxiters=None, ).mask From ea862f7425aadb5239a1a27fc2c2db2e5381a9fb Mon Sep 17 00:00:00 2001 From: Dougal Dobie Date: Thu, 29 Aug 2024 14:55:18 +1000 Subject: [PATCH 4/7] Added compress_quant argument to core.py --- vast_post_processing/core.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/vast_post_processing/core.py b/vast_post_processing/core.py index 4b163d7..635273a 100644 --- a/vast_post_processing/core.py +++ b/vast_post_processing/core.py @@ -147,6 +147,7 @@ def setup_configuration( crop_size: Optional[float] = None, create_moc: Optional[bool] = None, compress: Optional[bool] = None, + compress_quant: Optional[int] = None, directory_suffix: Optional[str] = None, cat_extension: Optional[str] = None, fits_extension: Optional[str] = None, @@ -161,6 +162,7 @@ def setup_configuration( u.Quantity, bool, bool, + int, str, str, str, @@ -192,6 +194,8 @@ def setup_configuration( Flag to create MOCs, by default None. compress : Optional[bool], optional Flag to compress files, by default None. + compress_quant : Optional[int], optional + Compression quantisation level to use, by default None. directory_suffix : Optional[str], optional Suffix to use for processed data directories (for example `STOKESI_IMAGES_PROCESSED`), by default None. @@ -239,6 +243,7 @@ def setup_configuration( "crop_size": crop_size, "create_moc": create_moc, "compress": compress, + "compress_quant": compress_quant, "directory_suffix": directory_suffix, "cat_extension": cat_extension, "fits_extension": fits_extension, @@ -468,6 +473,7 @@ def crop_image( corrected_fits: list[Union[fits.PrimaryHDU, fits.HDUList]], crop_size: u.Quantity, compress: bool, + compress_quant: int, overwrite: bool, verbose: bool, debug: bool, @@ -494,6 +500,8 @@ def crop_image( Angular size of crop to be applied. compress : bool Flag to compress image data. + compress_quant : int + Compression quantisation to use. overwrite : bool Flag to overwrite image data. verbose : bool @@ -544,7 +552,12 @@ def crop_image( cropped_hdu = crop.crop_hdu(image_hdu, field_centre, size=crop_size) # Compress image if requested - processed_hdu = compress_hdu(cropped_hdu) if compress else cropped_hdu + if compress: + processed_hdu = compress_hdu(cropped_hdu, + quantize_level=compress_quant + ) + else: + processed_hdu = cropped_hdu fitsutils.update_header_history(processed_hdu.header) if hdul is not None: @@ -729,6 +742,7 @@ def run( crop_size: Optional[float] = None, create_moc: Optional[bool] = None, compress: Optional[bool] = None, + compress_quant: Optional[int] = None, directory_suffix: Optional[str] = None, cat_extension: Optional[str] = None, fits_extension: Optional[str] = None, @@ -760,6 +774,8 @@ def run( Flag to create MOCs, by default None. compress : Optional[bool], optional Flag to compress files, by default None. + compress_quant : Optional[int], optional + Compression quantisation to use, by default None. directory_suffix : Optional[str], optional Suffix to use for processed data directories (for example `STOKESI_IMAGES_PROCESSED`), by default None. @@ -786,6 +802,7 @@ def run( crop_size, create_moc, compress, + compress_quant, directory_suffix, cat_extension, fits_extension, @@ -802,6 +819,7 @@ def run( crop_size=crop_size, create_moc=create_moc, compress=compress, + compress_quant=compress_quant, directory_suffix=directory_suffix, cat_extension=cat_extension, fits_extension=fits_extension, @@ -892,6 +910,7 @@ def run( verbose=verbose, debug=debug, compress=compress, + compress_quant=compress_quant, processed_dir_suffix=directory_suffix, ) if type(cropped_hdu) == fits.HDUList: From f8a380e2814e9dd5d00639c0e780910170ba2e7d Mon Sep 17 00:00:00 2001 From: Dougal Dobie Date: Thu, 29 Aug 2024 14:56:59 +1000 Subject: [PATCH 5/7] Add compress_quant argument to CLI --- vast_post_processing/cli/vast_post_processing.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vast_post_processing/cli/vast_post_processing.py b/vast_post_processing/cli/vast_post_processing.py index 8fda590..3b4f003 100644 --- a/vast_post_processing/cli/vast_post_processing.py +++ b/vast_post_processing/cli/vast_post_processing.py @@ -80,6 +80,9 @@ def main( compress: Optional[bool] = typer.Option( None, help=("Compress all processed FITS files") ), + compress_quant: Optional[int] = typer.Option( + None, help=("Compression quantisation to use, if compressing.") + ), directory_suffix: Optional[str] = typer.Option( None, help=("Suffix to use for processed data directories. For " "example, passing 'PROCESSED' results in images being " @@ -116,6 +119,7 @@ def main( crop_size, create_moc, compress, + compress_quant, directory_suffix, cat_extension, fits_extension, From 4e77b562f6199abb80d43a9d470e1bf61d235f17 Mon Sep 17 00:00:00 2001 From: Dougal Dobie Date: Thu, 29 Aug 2024 14:58:05 +1000 Subject: [PATCH 6/7] Add compress_quant argument to default_config.yaml --- vast_post_processing/data/default_config.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vast_post_processing/data/default_config.yaml b/vast_post_processing/data/default_config.yaml index 4a8c2b8..5e66051 100644 --- a/vast_post_processing/data/default_config.yaml +++ b/vast_post_processing/data/default_config.yaml @@ -30,6 +30,9 @@ create_moc: True # Flag to compress data following pipeline, by default True. compress: True +#Compression quantisation level to use, by default 1024. +compress_quant: 1024 + # Suffix to use for processed data directories # By default "PROCESSED" which results in e.g. `STOKESI_IMAGES_PROCESSED directory_suffix: "PROCESSED" From dabbf27fa485378178ff43691dc46d7e984e0193 Mon Sep 17 00:00:00 2001 From: Dougal Dobie Date: Thu, 29 Aug 2024 15:01:39 +1000 Subject: [PATCH 7/7] Fixed syntax --- vast_post_processing/data/default_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vast_post_processing/data/default_config.yaml b/vast_post_processing/data/default_config.yaml index 5e66051..18330fa 100644 --- a/vast_post_processing/data/default_config.yaml +++ b/vast_post_processing/data/default_config.yaml @@ -30,7 +30,7 @@ create_moc: True # Flag to compress data following pipeline, by default True. compress: True -#Compression quantisation level to use, by default 1024. +# Compression quantisation level to use, by default 1024. compress_quant: 1024 # Suffix to use for processed data directories