From e24107f34a67852208a554bcfbaabb74751660ef Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Thu, 5 Oct 2023 09:57:14 -0400 Subject: [PATCH 01/13] Migrate to no longer use SEP and use astropy Photutils instead. --- CHANGES.md | 4 + Dockerfile | 9 +- banzai/bpm.py | 2 +- banzai/photometry.py | 220 +++++++++++++++++++++---------------------- banzai/settings.py | 3 + docs/index.rst | 17 ++-- setup.cfg | 3 +- 7 files changed, 131 insertions(+), 127 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 53ecf9f0..e05e1e5f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +1.13.0 (2023-10-05) +------------------- +- Migrated photometry extraction to be done by astropy's photutils instead of SEP. + 1.12.0 (2023-08-10) ------------------- - Added the process_by_group keyword to stages to fix a bug that wouldn't allow grouping only by instrument diff --git a/Dockerfile b/Dockerfile index 5cbf2795..2d2e5119 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,8 @@ RUN conda install -y numpy pip scipy astropy pytest mock requests ipython covera && conda install -y -c conda-forge kombu=4.4.0 elasticsearch\<6.0.0,\>=5.0.0 pytest-astropy mysql-connector-python\ && conda clean -y --all -RUN pip install --no-cache-dir cython logutils lcogt_logging python-dateutil sqlalchemy\>=1.3.0b1 psycopg2-binary celery[redis]==4.3.0 \ - apscheduler ocs-ingester tenacity amqp==2.6.0 cosmic-conn +RUN pip install --no-cache-dir photutils bottleneck cython logutils lcogt_logging python-dateutil sqlalchemy\>=1.3.0b1 psycopg2-binary \ + celery[redis]==4.3.0 apscheduler ocs-ingester tenacity amqp==2.6.0 cosmic-conn RUN mkdir /home/archive && /usr/sbin/groupadd -g 10000 "domainusers" \ && /usr/sbin/useradd -g 10000 -d /home/archive -M -N -u 10087 archive \ @@ -13,11 +13,6 @@ RUN mkdir /home/archive && /usr/sbin/groupadd -g 10000 "domainusers" \ COPY --chown=10087:10000 . /lco/banzai -RUN apt-get -y update && apt-get -y install gcc && \ - pip install --no-cache-dir git+https://github.com/cmccully/sep.git@deblending /lco/banzai/ && \ - apt-get -y remove gcc && \ - apt-get autoclean && \ - rm -rf /var/lib/apt/lists/* USER archive diff --git a/banzai/bpm.py b/banzai/bpm.py index 473f3354..59d2ac14 100644 --- a/banzai/bpm.py +++ b/banzai/bpm.py @@ -39,6 +39,6 @@ def calibration_type(self): class SaturatedPixelFlagger(Stage): def do_stage(self, image): for image_extension in image.ccd_hdus: - image_extension.mask[image_extension.data > image_extension.saturate] |= 2 + image_extension.mask[image_extension.data >= image_extension.saturate] |= 2 return image diff --git a/banzai/photometry.py b/banzai/photometry.py index afb4eaec..b803a1e5 100755 --- a/banzai/photometry.py +++ b/banzai/photometry.py @@ -2,7 +2,6 @@ import numpy as np from astropy.table import Table -import sep from requests import HTTPError from banzai.utils import stats, array_utils @@ -11,10 +10,14 @@ from banzai.data import DataTable from banzai import logs +from photutils.background import Background2D from skimage import measure +from photutils.segmentation import make_2dgaussian_kernel, detect_sources, deblend_sources, SourceCatalog +from astropy.convolution import convolve +from astropy.convolution.kernels import CustomKernel + logger = logs.get_logger() -sep.set_sub_object_limit(int(1e6)) def radius_of_contour(contour, source): @@ -23,13 +26,49 @@ def radius_of_contour(contour, source): x_center = (source['xmax'] - source['xmin'] + 1) / 2.0 - 0.5 y_center = (source['ymax'] - source['ymin'] + 1) / 2.0 - 0.5 - return np.percentile(np.sqrt((x - x_center)**2.0 + (y - y_center)** 2.0), 90) + return np.percentile(np.sqrt((x - x_center)**2.0 + (y - y_center) ** 2.0), 90) + + +def flag_sources(sources, source_labels, segmentation_map, mask, flag, mask_value): + affected_sources = np.unique(segmentation_map[mask == mask_value]) + sources['flag'][np.in1d(source_labels, affected_sources)] |= flag + + +def flag_deblended(sources, catalog, segmentation_map, deblended_seg_map, flag_value=2): + # By default deblending appends labels instead of reassigning them so we can just use the + # extras in the deblended map + deblended_sources = np.unique(deblended_seg_map[deblended_seg_map > np.max(segmentation_map)]) + # Get the sources that were originally blended + original_blends = np.unique(segmentation_map[deblended_seg_map > np.max(segmentation_map)]) + deblended_sources = np.hstack([deblend_sources, original_blends]) + sources['flag'][np.in1d(catalog.labels, deblended_sources)] |= flag_value + + +def flag_edge_sources(image, sources, flag_value=8): + ny, nx = image.shape + # Check 4 points on the kron aperture, one on each side of the major and minor axis + minor_xmin = sources['x'] - sources['b'] * sources['kronrad'] * np.sin(np.deg2rad(sources['theta'])) + minor_xmax = sources['x'] + sources['b'] * sources['kronrad'] * np.sin(np.deg2rad(sources['theta'])) + minor_ymin = sources['y'] - sources['b'] * sources['kronrad'] * np.cos(np.deg2rad(sources['theta'])) + minor_ymax = sources['y'] + sources['b'] * sources['kronrad'] * np.cos(np.deg2rad(sources['theta'])) + major_ymin = sources['y'] - sources['a'] * sources['kronrad'] * np.sin(np.deg2rad(sources['theta'])) + major_ymax = sources['y'] + sources['a'] * sources['kronrad'] * np.sin(np.deg2rad(sources['theta'])) + major_xmin = sources['x'] - sources['a'] * sources['kronrad'] * np.cos(np.deg2rad(sources['theta'])) + major_xmax = sources['x'] + sources['a'] * sources['kronrad'] * np.cos(np.deg2rad(sources['theta'])) + + # Note we are already 1 indexed here + sources_off = np.logical_or(minor_xmin < 1, major_xmin < 1) + sources_off = np.logical_or(sources_off, minor_ymin < 1) + sources_off = np.logical_or(sources_off, major_ymin < 1) + sources_off = np.logical_or(sources_off, minor_xmax > nx) + sources_off = np.logical_or(sources_off, major_xmax > nx) + sources_off = np.logical_or(sources_off, minor_ymax > ny) + sources_off = np.logical_or(sources_off, major_ymax > ny) + sources[sources_off] |= flag_value class SourceDetector(Stage): - # Note that threshold is number of sigma, not an absolute number because we provide the error - # array to SEP. - threshold = 10.0 + threshold = 2.5 min_area = 9 def __init__(self, runtime_context): @@ -39,79 +78,75 @@ def do_stage(self, image): try: # Increase the internal buffer size in sep. This is most necessary for crowded fields. ny, nx = image.shape - sep.set_extract_pixstack(int(nx * ny - 1)) data = image.data.copy() error = image.uncertainty - mask = image.mask > 0 - # Fits can be backwards byte order, so fix that if need be and subtract - # the background - try: - bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3) - except ValueError: - data = data.byteswap(True).newbyteorder() - bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3) - bkg.subfrom(data) + # From what I can piece together, the background estimator makes a low resolution mesh set by box size + # (32, 32) here and then applies a filter to the low resolution image. The filter size is 3x3 here. + # The defaults we use here are a mesh creator is from source extractor which is a mode estimator. + # The default filter that works on the mesh image is a median filter. + bkg = Background2D(data, (32, 32), filter_size=(3, 3)) + data -= bkg.background + + # Convolve the image with a 2D Guassian, but with the normalization SEP uses as + # that is correct. + # The default kernel used by Source Extractor is [[1,2,1], [2,4,2], [1,2,1]] + # The kernel corresponds to fwhm = 1.9 which we adopt here + kernel = make_2dgaussian_kernel(1.9, size=3) + convolved_data = convolve(data / (error * error), kernel) + + # We include the correct match filter normalization here that is not included in + # vanilla source extractor + kernel_squared = CustomKernel(kernel.array * kernel.array) + normalization = np.sqrt(convolve(1 / (error * error), kernel_squared)) + convolved_data /= normalization # Do an initial source detection - try: - sources = sep.extract(data, self.threshold, mask=mask, minarea=self.min_area, - err=error, deblend_cont=0.005) - except Exception: - logger.error(logs.format_exception(), image=image) - return image - - # Convert the detections into a table - sources = Table(sources) - - # We remove anything with a detection flag >= 8 - # This includes memory overflows and objects that are too close the edge - sources = sources[sources['flag'] < 8] + segmentation_map = detect_sources(convolved_data, self.threshold, npixels=self.min_area) + + # Note that nlevels here is DEBLEND_NTHRESH in source extractor which is 32 by default + deblended_seg_map = deblend_sources(convolved_data, segmentation_map, + npixels=self.min_area, nlevels=32, + contrast=0.005, progress_bar=False, + nproc=self.runtime_context.N_PHOT_CORES) + # Convert the segmentation map to a source catalog + catalog = SourceCatalog(data, deblended_seg_map, convolved_data=convolved_data, error=error, + background=bkg.background) + + sources = Table({'x': catalog.xcentroid + 1.0, 'y': catalog.ycentroid + 1.0, + 'xwin': catalog.xcentroid_win + 1.0, 'ywin': catalog.ycentroid_win + 1.0, + 'xpeak': catalog.maxval_xindex + 1, 'ypeak': catalog.maxval_yindex + 1, + 'peak': catalog.max_value, + 'a': catalog.semimajor_sigma.value, 'b': catalog.semiminor_sigma.value, + 'theta': catalog.orientation.to('deg').value, 'ellipticity': catalog.ellipticity.value, + 'kronrad': catalog.kron_radius.value, + 'flux': catalog.kron_flux, 'fluxerr': catalog, + 'x2': catalog.covar_sigx2.value, 'y2': catalog.covar_sigy2.value, + 'xy': catalog.covar_sigxy.value, + 'background': catalog.background_mean}) + + for r in range(1, 7): + radius_arcsec = r / image.pixel_scale + sources[f'fluxaper{r}'], sources[f'fluxerr{r}'] = catalog.circular_photometry(radius_arcsec) + + for r in [0.25, 0.5, 0.75]: + sources['fluxrad' + f'{r:.2f}'.lstrip("0.")] = catalog.fluxfrac_radius(r) + + sources['flag'] = 0 + + # Flag = 1 for sources with bad pixels + flag_sources(sources, catalog.labels, deblended_seg_map, image.mask, flag=1, mask_value=1) + # Flag = 2 for sources that are deblended + flag_deblended(sources, catalog, segmentation_map, deblended_seg_map, flag_value=2) + # Flag = 4 for sources that have saturated pixels + flag_sources(sources, catalog.labels, deblended_seg_map, image.mask, flag=4, mask_value=2) + # Flag = 8 if kron aperture falls off the image + # Flag = 16 if source has cosmic ray pixels + flag_sources(sources, catalog.labels, deblended_seg_map, image.mask, flag=16, mask_value=8) sources = array_utils.prune_nans_from_table(sources) - # Calculate the ellipticity - sources['ellipticity'] = 1.0 - (sources['b'] / sources['a']) - - # Fix any value of theta that are invalid due to floating point rounding - # -pi / 2 < theta < pi / 2 - sources['theta'][sources['theta'] > (np.pi / 2.0)] -= np.pi - sources['theta'][sources['theta'] < (-np.pi / 2.0)] += np.pi - - # Calculate the kron radius - kronrad, krflag = sep.kron_radius(data, sources['x'], sources['y'], - sources['a'], sources['b'], - sources['theta'], 6.0) - sources['flag'] |= krflag - sources['kronrad'] = kronrad - - # Calcuate the equivilent of flux_auto - flux, fluxerr, flag = sep.sum_ellipse(data, sources['x'], sources['y'], - sources['a'], sources['b'], - np.pi / 2.0, 2.5 * kronrad, - subpix=1, err=error) - sources['flux'] = flux - sources['fluxerr'] = fluxerr - sources['flag'] |= flag - - # Do circular aperture photometry for diameters of 1" to 6" - for diameter in [1, 2, 3, 4, 5, 6]: - flux, fluxerr, flag = sep.sum_circle(data, sources['x'], sources['y'], - diameter / 2.0 / image.pixel_scale, gain=1.0, err=error) - sources['fluxaper{0}'.format(diameter)] = flux - sources['fluxerr{0}'.format(diameter)] = fluxerr - sources['flag'] |= flag - - # Measure the flux profile - flux_radii, flag = sep.flux_radius(data, sources['x'], sources['y'], - 6.0 * sources['a'], [0.25, 0.5, 0.75], - normflux=sources['flux'], subpix=5) - sources['flag'] |= flag - sources['fluxrad25'] = flux_radii[:, 0] - sources['fluxrad50'] = flux_radii[:, 1] - sources['fluxrad75'] = flux_radii[:, 2] - # Cut individual bright pixels. Often cosmic rays sources = sources[sources['fluxrad50'] > 0.5] @@ -130,45 +165,6 @@ def do_stage(self, image): contour_radii = [radius_of_contour(contour, source) for contour in contours] source[keyword] = 2.0 * np.nanmax(contour_radii) - # Calculate the windowed positions - sig = 2.0 / 2.35 * sources['fwhm'] - xwin, ywin, flag = sep.winpos(data, sources['x'], sources['y'], sig) - sources['flag'] |= flag - sources['xwin'] = xwin - sources['ywin'] = ywin - - # Calculate the average background at each source - bkgflux, fluxerr, flag = sep.sum_ellipse(bkg.back(), sources['x'], sources['y'], - sources['a'], sources['b'], np.pi / 2.0, - 2.5 * sources['kronrad'], subpix=1) - # masksum, fluxerr, flag = sep.sum_ellipse(mask, sources['x'], sources['y'], - # sources['a'], sources['b'], np.pi / 2.0, - # 2.5 * kronrad, subpix=1) - - background_area = (2.5 * sources['kronrad']) ** 2.0 * sources['a'] * sources['b'] * np.pi # - masksum - sources['background'] = bkgflux - sources['background'][background_area > 0] /= background_area[background_area > 0] - # Update the catalog to match fits convention instead of python array convention - sources['x'] += 1.0 - sources['y'] += 1.0 - - sources['xpeak'] += 1 - sources['ypeak'] += 1 - - sources['xwin'] += 1.0 - sources['ywin'] += 1.0 - - sources['theta'] = np.degrees(sources['theta']) - - catalog = sources['x', 'y', 'xwin', 'ywin', 'xpeak', 'ypeak', - 'flux', 'fluxerr', 'peak', 'fluxaper1', 'fluxerr1', - 'fluxaper2', 'fluxerr2', 'fluxaper3', 'fluxerr3', - 'fluxaper4', 'fluxerr4', 'fluxaper5', 'fluxerr5', - 'fluxaper6', 'fluxerr6', 'background', 'fwhm', 'fwtm', - 'a', 'b', 'theta', 'kronrad', 'ellipticity', - 'fluxrad25', 'fluxrad50', 'fluxrad75', - 'x2', 'y2', 'xy', 'flag'] - # Add the units and description to the catalogs catalog['x'].unit = 'pixel' catalog['x'].description = 'X coordinate of the object' @@ -227,15 +223,15 @@ def do_stage(self, image): catalog.reverse() # Save some background statistics in the header - mean_background = stats.sigma_clipped_mean(bkg.back(), 5.0) + mean_background = stats.sigma_clipped_mean(bkg.background(), 5.0) image.meta['L1MEAN'] = (mean_background, '[counts] Sigma clipped mean of frame background') - median_background = np.median(bkg.back()) + median_background = np.median(bkg.background()) image.meta['L1MEDIAN'] = (median_background, '[counts] Median of frame background') - std_background = stats.robust_standard_deviation(bkg.back()) + std_background = stats.robust_standard_deviation(bkg.background()) image.meta['L1SIGMA'] = (std_background, '[counts] Robust std dev of frame background') diff --git a/banzai/settings.py b/banzai/settings.py index d85197b1..f674657e 100644 --- a/banzai/settings.py +++ b/banzai/settings.py @@ -156,3 +156,6 @@ CELERY_TASK_QUEUE_NAME = os.getenv('CELERY_TASK_QUEUE_NAME', 'celery') REFERENCE_CATALOG_URL = os.getenv('REFERENCE_CATALOG_URL', 'http://phot-catalog.lco.gtn/') + +# Number of cores to use for photometry deblending +N_PHOT_CORES = 4 diff --git a/docs/index.rst b/docs/index.rst index ac03b777..79ed3331 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -107,20 +107,18 @@ our models are 94% complete for our training data. Pixels flagged as cosmic-ray Source Detection ================ -Source detection uses the "Source Extraction in Python" (SEP; https://github.com/kbarbary/sep). -This is similar to Source Extractor, but is written purely in Python and Cython. This allows more -customization. +Source detection uses the "astropy.photutils" image segmentation. +This is similar to Source Extractor, but is written purely in Python. This allows more customization than the original SExtractor. We estimate the background by taking a 3x3 median filter of the image and the doing a 32x32 block average of the image. -We use the default match filter for source detection that is provided by SEP. +We use the default match filter for source detection that is provided in Source Extractor. We include the proper match filter normalization though so the extraction will be slightly different. We do aperture photometry using an elliptical aperture that is set by 2.5 times the Kron radius. This produces approximately the same results as ``FLUX_AUTO`` from SExtractor. -We set the source detection limit at 3 times the global rms of the image. ``MINAREA`` is set to 5, -the default. This should minimize false detections, but may miss the faintest sources. +We set the source detection limit at 2.5 times the uncertainty image. ``MINAREA`` is set to 9. This should minimize false detections, but may miss the faintest sources. To assess the image quality, we estimate the full-width half maximum (FWHM) of the stars in the image. We reject any sources that have a FWHM of less than a pixel to ensure that they do not bias our results. The PSFs for LCO are @@ -132,6 +130,13 @@ estimate of the FWHM. This ensures that we do not underestimate the FWHM when de we take the robust standard deviation (see below) to estimate the overall FWHM of the image. This value is recorded in the header under the L1FWHM keyword. +Flags are as follows: +- 1: Source has bad pixels in the image segmentation +- 2: Object is deblended +- 4: Source has saturated pixels in the image segmentation +- 8: Source kron aperture falls off the image +- 16: Source has cosmic ray pixels in the image segmentation + Astrometry ========== diff --git a/setup.cfg b/setup.cfg index 3a8e574f..1e54b8bc 100755 --- a/setup.cfg +++ b/setup.cfg @@ -109,7 +109,8 @@ install_requires = cython mysql-connector-python lcogt_logging==0.3.2 - sep + photutils + bottleneck kombu==4.4.0 amqp==2.6.0 requests From 101fad3d1b13223fcce887157eab7dfaf78829fd Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Wed, 11 Oct 2023 09:04:06 -0700 Subject: [PATCH 02/13] Add docker build for banzai and push to ghcr --- .github/workflows/build-docker.yml | 58 ++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 .github/workflows/build-docker.yml diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml new file mode 100644 index 00000000..2258a71b --- /dev/null +++ b/.github/workflows/build-docker.yml @@ -0,0 +1,58 @@ +name: Docker Image + +concurrency: ${{ github.workflow }}-${{ github.ref }} + +on: + push: + branches: + - "*" + tags: + - "*" + pull_request: + branches: + - master + +jobs: + build-push: + + runs-on: ubuntu-latest + + permissions: + contents: write + packages: write + + env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + + steps: + - name: "Checkout" + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + install: true + + - name: Login to Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and also push Dockerimage (only build on PRs) + id: build-and-push + uses: docker/build-push-action@v3 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} From 090453e81c1e9b461a882e33cda114c06073d761 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Wed, 11 Oct 2023 12:30:07 -0400 Subject: [PATCH 03/13] Fix to actually install banzai --- Dockerfile | 5 +++++ banzai/tests/e2e-k8s.yaml | 7 +++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2d2e5119..3d2992db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,11 @@ RUN mkdir /home/archive && /usr/sbin/groupadd -g 10000 "domainusers" \ COPY --chown=10087:10000 . /lco/banzai +RUN apt-get -y update && apt-get -y install gcc && \ + pip install --no-cache-dir /lco/banzai/ && \ + apt-get -y remove gcc && \ + apt-get autoclean && \ + rm -rf /var/lib/apt/lists/* USER archive diff --git a/banzai/tests/e2e-k8s.yaml b/banzai/tests/e2e-k8s.yaml index 9fcfa266..39bf9129 100644 --- a/banzai/tests/e2e-k8s.yaml +++ b/banzai/tests/e2e-k8s.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: Pod metadata: name: banzai-e2e-test - namespace: build labels: app.kubernetes.io/name: banzai spec: @@ -80,7 +79,7 @@ spec: periodSeconds: 1 timeoutSeconds: 10 - name: banzai-celery-workers - image: @BANZAI_IMAGE@ + image: docker.lco.global/banzai:1.11.0-34-ge24107f3 imagePullPolicy: IfNotPresent volumeMounts: - name: banzai-data @@ -144,7 +143,7 @@ spec: cpu: 8 memory: 8Gi - name: banzai-celery-beat - image: @BANZAI_IMAGE@ + image: docker.lco.global/banzai:1.11.0-34-ge24107f3 imagePullPolicy: IfNotPresent volumeMounts: - name: banzai-data @@ -170,7 +169,7 @@ spec: cpu: 1 memory: 1Gi - name: banzai-listener - image: @BANZAI_IMAGE@ + image: docker.lco.global/banzai:1.11.0-34-ge24107f3 imagePullPolicy: IfNotPresent volumeMounts: - name: banzai-data From 7b3c5d4c96c767833bf71bd51421e77cca9f95d8 Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Wed, 11 Oct 2023 09:54:56 -0700 Subject: [PATCH 04/13] Always push docker image --- .github/workflows/build-docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 2258a71b..b9285031 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -48,11 +48,11 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - name: Build and also push Dockerimage (only build on PRs) + - name: Build and also push Dockerimage id: build-and-push uses: docker/build-push-action@v3 with: context: . - push: ${{ github.event_name != 'pull_request' }} + push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 0c0695c55ebd3a028ce2fd80a2a3e05207aeddcb Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Thu, 12 Oct 2023 10:36:44 -0400 Subject: [PATCH 05/13] Added pinned versions to docker build. Should make it faster. --- Dockerfile | 28 ++-- environment.yaml | 323 +++++++++++++++++++++++++++++++++++++++++++++++ setup.cfg | 12 +- 3 files changed, 342 insertions(+), 21 deletions(-) create mode 100644 environment.yaml diff --git a/Dockerfile b/Dockerfile index 3d2992db..c22933af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,26 +1,24 @@ -FROM continuumio/miniconda3:4.10.3 +FROM continuumio/miniconda3:23.5.2-0 -RUN conda install -y numpy pip scipy astropy pytest mock requests ipython coverage pyyaml\ - && conda install -y -c conda-forge kombu=4.4.0 elasticsearch\<6.0.0,\>=5.0.0 pytest-astropy mysql-connector-python\ - && conda clean -y --all - -RUN pip install --no-cache-dir photutils bottleneck cython logutils lcogt_logging python-dateutil sqlalchemy\>=1.3.0b1 psycopg2-binary \ - celery[redis]==4.3.0 apscheduler ocs-ingester tenacity amqp==2.6.0 cosmic-conn +# In principle I could remove the gcc to shrink the image, but pytorch is already so large it doesn't make much difference +RUN apt-get -y update && apt-get -y install gcc && \ + apt-get autoclean && \ + rm -rf /var/lib/apt/lists/* RUN mkdir /home/archive && /usr/sbin/groupadd -g 10000 "domainusers" \ && /usr/sbin/useradd -g 10000 -d /home/archive -M -N -u 10087 archive \ && chown -R archive:domainusers /home/archive -COPY --chown=10087:10000 . /lco/banzai - -RUN apt-get -y update && apt-get -y install gcc && \ - pip install --no-cache-dir /lco/banzai/ && \ - apt-get -y remove gcc && \ - apt-get autoclean && \ - rm -rf /var/lib/apt/lists/* - USER archive ENV HOME /home/archive WORKDIR /home/archive + +COPY environment.yaml . + +RUN . /opt/conda/etc/profile.d/conda.sh && conda env create -p /home/archive/envs/banzai -f environment.yaml --solver=libmamba && conda activate /home/archive/envs/banzai + +COPY --chown=10087:10000 . /lco/banzai + +RUN /home/archive/envs/banzai/bin/pip install --no-cache-dir /lco/banzai/ diff --git a/environment.yaml b/environment.yaml new file mode 100644 index 00000000..0fad571b --- /dev/null +++ b/environment.yaml @@ -0,0 +1,323 @@ +channels: + - pytorch + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_kmp_llvm + - affine=2.4.0=pyhd8ed1ab_0 + - amqp=2.6.1=pyh9f0ad1d_0 + - anyio=4.0.0=pyhd8ed1ab_0 + - aom=3.6.1=h59595ed_0 + - apscheduler=3.10.4=pyhd8ed1ab_0 + - asdf=2.15.1=pyhd8ed1ab_0 + - asdf-astropy=0.4.0=pyhd8ed1ab_1 + - asdf-coordinates-schemas=0.2.0=pyhd8ed1ab_1 + - asdf-standard=1.0.3=pyhd8ed1ab_0 + - asdf-transform-schemas=0.3.0=pyhd8ed1ab_0 + - asdf-unit-schemas=0.1.0=pyhd8ed1ab_0 + - asdf-wcs-schemas=0.1.1=pyhd8ed1ab_0 + - astropy=5.3.4=py311h1f0f07a_2 + - asttokens=2.4.0=pyhd8ed1ab_0 + - attrs=23.1.0=pyh71513ae_1 + - backcall=0.2.0=pyh9f0ad1d_0 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 + - backports.zoneinfo=0.2.1=py311h38be061_8 + - billiard=3.6.4.0=py311hd4cff14_3 + - blas=2.116=mkl + - blas-devel=3.9.0=16_linux64_mkl + - blosc=1.21.5=h0f2a231_0 + - bottleneck=1.3.7=py311h1f0f07a_1 + - brotli=1.1.0=hd590300_1 + - brotli-bin=1.1.0=hd590300_1 + - brotli-python=1.1.0=py311hb755f60_1 + - brunsli=0.1=h9c3ff4c_0 + - bzip2=1.0.8=h7f98852_4 + - c-ares=1.20.1=hd590300_0 + - c-blosc2=2.10.5=hb4ffafa_0 + - ca-certificates=2023.7.22=hbcca054_0 + - cairo=1.18.0=h3faef2a_0 + - celery=4.4.0=py_0 + - certifi=2023.7.22=pyhd8ed1ab_0 + - cffi=1.16.0=py311hb3a22ac_0 + - cfitsio=4.3.0=hbdc6101_0 + - charls=2.4.2=h59595ed_0 + - charset-normalizer=3.3.0=pyhd8ed1ab_0 + - click=8.1.7=unix_pyh707e725_0 + - click-plugins=1.1.1=py_0 + - cligj=0.7.2=pyhd8ed1ab_1 + - colorama=0.4.6=pyhd8ed1ab_0 + - contourpy=1.1.1=py311h9547e67_1 + - coverage=7.3.2=py311h459d7ec_0 + - cryptography=41.0.4=py311h63ff55d_0 + - cycler=0.12.1=pyhd8ed1ab_0 + - cython=0.29.36=py311hb755f60_1 + - dav1d=1.2.1=hd590300_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - dnspython=2.4.2=pyhd8ed1ab_0 + - elasticsearch=5.5.3=pyh9f0ad1d_0 + - emcee=3.1.4=pyhd8ed1ab_0 + - exceptiongroup=1.1.3=pyhd8ed1ab_0 + - executing=1.2.0=pyhd8ed1ab_0 + - expat=2.5.0=hcb278e6_1 + - filelock=3.12.4=pyhd8ed1ab_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=hab24e00_0 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.43.1=py311h459d7ec_0 + - freetype=2.12.1=h267a509_2 + - freexl=2.0.0=h743c826_0 + - geos=3.12.0=h59595ed_0 + - geotiff=1.7.1=hee599c5_13 + - gettext=0.21.1=h27087fc_0 + - giflib=5.2.1=h0b41bf4_3 + - gmp=6.2.1=h58526e2_0 + - gmpy2=2.1.2=py311h6a5fa03_1 + - greenlet=3.0.0=py311hb755f60_1 + - gwcs=0.19.0=pyhd8ed1ab_0 + - h11=0.14.0=pyhd8ed1ab_0 + - h2=4.1.0=pyhd8ed1ab_0 + - hdf4=4.2.15=h501b40f_6 + - hdf5=1.14.2=nompi_h4f84152_100 + - hpack=4.0.0=pyh9f0ad1d_0 + - httpcore=1.0.0=pyhd8ed1ab_0 + - hyperframe=6.0.1=pyhd8ed1ab_0 + - hypothesis=6.87.3=pyha770c72_0 + - icu=73.2=h59595ed_0 + - idna=3.4=pyhd8ed1ab_0 + - imagecodecs=2023.9.18=py311h9b38416_0 + - imageio=2.31.5=pyh8c1a49c_0 + - importlib-metadata=6.8.0=pyha770c72_0 + - importlib-resources=6.1.0=pyhd8ed1ab_0 + - importlib_resources=6.1.0=pyhd8ed1ab_0 + - iniconfig=2.0.0=pyhd8ed1ab_0 + - ipython=8.16.1=pyh0d859eb_0 + - jedi=0.19.1=pyhd8ed1ab_0 + - jinja2=3.1.2=pyhd8ed1ab_1 + - jmespath=1.0.1=pyhd8ed1ab_0 + - joblib=1.3.2=pyhd8ed1ab_0 + - json-c=0.17=h7ab15ed_0 + - jsonschema=4.19.1=pyhd8ed1ab_0 + - jsonschema-specifications=2023.7.1=pyhd8ed1ab_0 + - jxrlib=1.1=h7f98852_2 + - kealib=1.5.2=hcd42e92_1 + - keyutils=1.6.1=h166bdaf_0 + - kiwisolver=1.4.5=py311h9547e67_1 + - krb5=1.21.2=h659d440_0 + - lazy_loader=0.3=pyhd8ed1ab_0 + - lcms2=2.15=h7f713cb_2 + - ld_impl_linux-64=2.40=h41732ed_0 + - lerc=4.0.0=h27087fc_0 + - libaec=1.1.2=h59595ed_1 + - libarchive=3.7.2=h039dbb9_0 + - libavif16=1.0.1=h87da1f6_2 + - libblas=3.9.0=16_linux64_mkl + - libboost-headers=1.82.0=ha770c72_6 + - libbrotlicommon=1.1.0=hd590300_1 + - libbrotlidec=1.1.0=hd590300_1 + - libbrotlienc=1.1.0=hd590300_1 + - libcblas=3.9.0=16_linux64_mkl + - libcurl=8.4.0=hca28451_0 + - libdeflate=1.19=hd590300_0 + - libedit=3.1.20191231=he28a2e2_2 + - libev=4.33=h516909a_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.2=h7f98852_5 + - libgcc-ng=13.2.0=h807b86a_2 + - libgdal=3.7.2=h3aa23ec_3 + - libgfortran-ng=13.2.0=h69a702a_2 + - libgfortran5=13.2.0=ha4646dd_2 + - libglib=2.78.0=hebfc3b9_0 + - libhwloc=2.9.3=default_h554bfaf_1009 + - libiconv=1.17=h166bdaf_0 + - libjpeg-turbo=2.1.5.1=hd590300_1 + - libkml=1.3.0=h01aab08_1018 + - liblapack=3.9.0=16_linux64_mkl + - liblapacke=3.9.0=16_linux64_mkl + - libnetcdf=4.9.2=nompi_h80fb2b6_112 + - libnghttp2=1.52.0=h61bc06f_0 + - libnsl=2.0.0=hd590300_1 + - libpng=1.6.39=h753d276_0 + - libpq=15.4=hfc447b1_2 + - libprotobuf=3.20.3=h3eb15da_0 + - librttopo=1.1.0=hb58d41b_14 + - libspatialite=5.1.0=h090f1da_0 + - libsqlite=3.43.2=h2797004_0 + - libssh2=1.11.0=h0841786_0 + - libstdcxx-ng=13.2.0=h7e041cc_2 + - libtiff=4.6.0=h29866fb_1 + - libuuid=2.38.1=h0b41bf4_0 + - libwebp-base=1.3.2=hd590300_0 + - libxcb=1.15=h0b41bf4_0 + - libxml2=2.11.5=h232c23b_1 + - libzip=1.10.1=h2629f0a_3 + - libzlib=1.2.13=hd590300_5 + - libzopfli=1.0.3=h9c3ff4c_0 + - llvm-openmp=15.0.7=h0cdce71_0 + - lz4-c=1.9.4=hcb278e6_0 + - lzo=2.10=h516909a_1000 + - markupsafe=2.1.3=py311h459d7ec_1 + - matplotlib-base=3.8.0=py311h54ef318_2 + - matplotlib-inline=0.1.6=pyhd8ed1ab_0 + - minizip=4.0.1=h0ab5242_5 + - mkl=2022.1.0=h84fe81f_915 + - mkl-devel=2022.1.0=ha770c72_916 + - mkl-include=2022.1.0=h84fe81f_915 + - mock=5.1.0=pyhd8ed1ab_0 + - mpc=1.3.1=hfe3b2da_0 + - mpfr=4.2.0=hb012696_0 + - mpmath=1.3.0=pyhd8ed1ab_0 + - munkres=1.1.4=pyh9f0ad1d_0 + - mysql-common=8.0.33=hf1915f5_5 + - mysql-connector-python=8.0.32=py311h381d6c5_0 + - mysql-libs=8.0.33=hca2cd23_5 + - ncurses=6.4=hcb278e6_0 + - networkx=3.1=pyhd8ed1ab_0 + - nspr=4.35=h27087fc_0 + - nss=3.94=h1d7d5a4_0 + - numpy=1.23.5=py311h7d28db0_0 + - openjpeg=2.5.0=h488ebb8_3 + - openssl=3.1.3=hd590300_0 + - packaging=23.2=pyhd8ed1ab_0 + - parso=0.8.3=pyhd8ed1ab_0 + - pcre2=10.40=hc3806b6_0 + - pexpect=4.8.0=pyh1a96a4e_2 + - photutils=1.9.0=py311h459d7ec_0 + - pickleshare=0.7.5=py_1003 + - pillow=10.0.1=py311h8aef010_1 + - pip=23.2.1=pyhd8ed1ab_0 + - pixman=0.42.2=h59595ed_0 + - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1 + - pluggy=1.3.0=pyhd8ed1ab_0 + - poppler=23.08.0=hf2349cb_2 + - poppler-data=0.4.12=hd8ed1ab_0 + - postgresql=15.4=h8972f4a_2 + - proj=9.3.0=h1d62c97_1 + - prompt-toolkit=3.0.39=pyha770c72_0 + - prompt_toolkit=3.0.39=hd8ed1ab_0 + - protobuf=3.20.3=py311hcafe171_1 + - psutil=5.9.5=py311h459d7ec_1 + - psycopg2=2.9.7=py311h68d4568_0 + - psycopg2-binary=2.9.7=pyhd8ed1ab_0 + - pthread-stubs=0.4=h36c2ea0_1001 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.2=pyhd8ed1ab_0 + - pycparser=2.21=pyhd8ed1ab_0 + - pyerfa=2.0.0.3=py311h1f0f07a_1 + - pygments=2.16.1=pyhd8ed1ab_0 + - pyparsing=3.1.1=pyhd8ed1ab_0 + - pysocks=1.7.1=pyha2e5f31_6 + - pytest=7.4.2=pyhd8ed1ab_0 + - pytest-arraydiff=0.5.0=pyhd8ed1ab_0 + - pytest-astropy=0.10.0=pyhd8ed1ab_0 + - pytest-astropy-header=0.2.2=pyhd8ed1ab_0 + - pytest-cov=4.1.0=pyhd8ed1ab_0 + - pytest-doctestplus=1.0.0=pyhd8ed1ab_0 + - pytest-filter-subpackage=0.1.2=pyhd8ed1ab_0 + - pytest-mock=3.11.1=pyhd8ed1ab_0 + - pytest-openfiles=0.5.0=py_0 + - pytest-remotedata=0.4.1=pyhd8ed1ab_0 + - python=3.11.6=hab00c5b_0_cpython + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python_abi=3.11=4_cp311 + - pytorch=2.1.0=py3.11_cpu_0 + - pytorch-mutex=1.0=cpu + - pytz=2023.3.post1=pyhd8ed1ab_0 + - pywavelets=1.4.1=py311h1f0f07a_1 + - pyyaml=6.0.1=py311h459d7ec_1 + - rasterio=1.3.8=py311h40fbdff_3 + - rav1e=0.6.6=he8a937b_2 + - readline=8.2=h8228510_1 + - referencing=0.30.2=pyhd8ed1ab_0 + - requests=2.31.0=pyhd8ed1ab_0 + - rpds-py=0.10.4=py311h46250e7_0 + - scikit-image=0.22.0=py311h320fe9a_2 + - scikit-learn=1.3.1=py311hc009520_1 + - scipy=1.11.3=py311h64a7726_1 + - semantic_version=2.10.0=pyhd8ed1ab_0 + - sep=1.2.1=py311h1f0f07a_2 + - setuptools=68.2.2=pyhd8ed1ab_0 + - shapely=2.0.1=py311he06c224_3 + - simplejson=3.19.2=py311h459d7ec_0 + - six=1.16.0=pyh6c4a22f_0 + - snappy=1.1.10=h9fff704_0 + - sniffio=1.3.0=pyhd8ed1ab_0 + - snuggs=1.4.7=py_0 + - sortedcontainers=2.4.0=pyhd8ed1ab_0 + - sqlalchemy=2.0.21=py311h459d7ec_1 + - sqlite=3.43.2=h2c6b66d_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - svt-av1=1.7.0=h59595ed_0 + - sympy=1.12=pypyh9d50eac_103 + - tbb=2021.10.0=h00ab1b0_1 + - tenacity=8.2.3=pyhd8ed1ab_0 + - threadpoolctl=3.2.0=pyha21a80b_0 + - tifffile=2023.9.26=pyhd8ed1ab_0 + - tiledb=2.16.3=hf0b6e87_3 + - tk=8.6.13=h2797004_0 + - toml=0.10.2=pyhd8ed1ab_0 + - tomli=2.0.1=pyhd8ed1ab_0 + - tqdm=4.66.1=pyhd8ed1ab_0 + - traitlets=5.11.2=pyhd8ed1ab_0 + - typing-extensions=4.8.0=hd8ed1ab_0 + - typing_extensions=4.8.0=pyha770c72_0 + - tzcode=2023c=h0b41bf4_0 + - tzdata=2023c=h71feb2d_0 + - tzlocal=5.1=py311h38be061_0 + - uriparser=0.9.7=hcb278e6_1 + - wcwidth=0.2.8=pyhd8ed1ab_0 + - wheel=0.41.2=pyhd8ed1ab_0 + - xerces-c=3.2.4=hac6953d_3 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.6=h8ee46fc_0 + - xorg-libxau=1.0.11=hd590300_0 + - xorg-libxdmcp=1.1.3=h7f98852_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xz=5.2.6=h166bdaf_0 + - yaml=0.2.5=h7f98852_2 + - zfp=1.0.0=h59595ed_4 + - zipp=3.17.0=pyhd8ed1ab_0 + - zlib=1.2.13=hd590300_5 + - zlib-ng=2.0.7=h0b41bf4_0 + - zstd=1.5.5=hfc55251_0 + - pip: + - asciitree==0.3.3 + - astropy-healpix==1.0.0 + - boto3==1.28.62 + - botocore==1.31.62 + - cloudpickle==2.2.1 + - cosmic-conn==0.4.1 + - dask==2023.9.3 + - fasteners==0.19 + - fsspec==2023.9.2 + - kombu==4.6.11 + - lcogt-logging==0.3.2 + - locket==1.0.0 + - logutils==0.3.5 + - numcodecs==0.12.0 + - ocs-archive==0.2.10 + - ocs-ingester==3.0.5 + - opensearch-py==1.1.0 + - opentsdb-http-client==0.2.0 + - opentsdb-python-metrics==0.2.0 + - partd==1.4.1 + - pretty-errors==1.2.25 + - redis==5.0.1 + - reproject==0.12.0 + - s3transfer==0.7.0 + - toolz==0.12.0 + - urllib3==1.26.17 + - vine==1.3.0 + - zarr==2.16.1 +prefix: /root/micromamba/envs/banzai diff --git a/setup.cfg b/setup.cfg index 1e54b8bc..1fe5c641 100755 --- a/setup.cfg +++ b/setup.cfg @@ -108,21 +108,21 @@ install_requires = numpy<1.24 cython mysql-connector-python - lcogt_logging==0.3.2 + lcogt_logging photutils bottleneck - kombu==4.4.0 - amqp==2.6.0 + kombu + amqp requests - opensearch-py==1.0.0 + opensearch-py>=1,<2 pytest>=4.0 pyyaml psycopg2-binary - celery[redis]==4.3.1 + celery[redis]>=4.3.1,<5 apscheduler python-dateutil ocs_ingester>=3.0.4,<4.0.0 - tenacity==6.0.0 + tenacity>=8,<=9 python-dateutil emcee scikit-image From 922078b0fa1b0102f1e367b547a4d05fbf74a626 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Fri, 13 Oct 2023 17:12:10 -0400 Subject: [PATCH 06/13] Finally got a working docker build. --- Dockerfile | 4 +- banzai/tests/e2e-k8s.yaml | 8 +-- environment.yaml | 141 ++++++++++++++++++-------------------- 3 files changed, 72 insertions(+), 81 deletions(-) diff --git a/Dockerfile b/Dockerfile index c22933af..947eda52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,8 +17,10 @@ WORKDIR /home/archive COPY environment.yaml . -RUN . /opt/conda/etc/profile.d/conda.sh && conda env create -p /home/archive/envs/banzai -f environment.yaml --solver=libmamba && conda activate /home/archive/envs/banzai +RUN . /opt/conda/etc/profile.d/conda.sh && conda env create -p /home/archive/envs/banzai -f environment.yaml --solver=libmamba COPY --chown=10087:10000 . /lco/banzai +ENV PATH /home/archive/envs/banzai/bin:$PATH + RUN /home/archive/envs/banzai/bin/pip install --no-cache-dir /lco/banzai/ diff --git a/banzai/tests/e2e-k8s.yaml b/banzai/tests/e2e-k8s.yaml index 39bf9129..02d9ea7d 100644 --- a/banzai/tests/e2e-k8s.yaml +++ b/banzai/tests/e2e-k8s.yaml @@ -61,7 +61,7 @@ spec: initialDelaySeconds: 5 periodSeconds: 1 - name: banzai-fits-exchange - image: rabbitmq:3.7.9 + image: rabbitmq:3.12.3 imagePullPolicy: IfNotPresent resources: requests: @@ -79,7 +79,7 @@ spec: periodSeconds: 1 timeoutSeconds: 10 - name: banzai-celery-workers - image: docker.lco.global/banzai:1.11.0-34-ge24107f3 + image: @BANZAI_IMAGE@ imagePullPolicy: IfNotPresent volumeMounts: - name: banzai-data @@ -143,7 +143,7 @@ spec: cpu: 8 memory: 8Gi - name: banzai-celery-beat - image: docker.lco.global/banzai:1.11.0-34-ge24107f3 + image: @BANZAI_IMAGE@ imagePullPolicy: IfNotPresent volumeMounts: - name: banzai-data @@ -169,7 +169,7 @@ spec: cpu: 1 memory: 1Gi - name: banzai-listener - image: docker.lco.global/banzai:1.11.0-34-ge24107f3 + image: @BANZAI_IMAGE@ imagePullPolicy: IfNotPresent volumeMounts: - name: banzai-data diff --git a/environment.yaml b/environment.yaml index 0fad571b..76094ea8 100644 --- a/environment.yaml +++ b/environment.yaml @@ -1,10 +1,11 @@ channels: - pytorch - conda-forge + - astropy - defaults dependencies: - _libgcc_mutex=0.1=conda_forge - - _openmp_mutex=4.5=2_kmp_llvm + - _openmp_mutex=4.5=2_gnu - affine=2.4.0=pyhd8ed1ab_0 - amqp=2.6.1=pyh9f0ad1d_0 - anyio=4.0.0=pyhd8ed1ab_0 @@ -17,21 +18,19 @@ dependencies: - asdf-transform-schemas=0.3.0=pyhd8ed1ab_0 - asdf-unit-schemas=0.1.0=pyhd8ed1ab_0 - asdf-wcs-schemas=0.1.1=pyhd8ed1ab_0 - - astropy=5.3.4=py311h1f0f07a_2 + - astropy=5.3.4=py310h1f7b6fc_2 - asttokens=2.4.0=pyhd8ed1ab_0 - attrs=23.1.0=pyh71513ae_1 - backcall=0.2.0=pyh9f0ad1d_0 - backports=1.0=pyhd8ed1ab_3 - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 - - backports.zoneinfo=0.2.1=py311h38be061_8 - - billiard=3.6.4.0=py311hd4cff14_3 - - blas=2.116=mkl - - blas-devel=3.9.0=16_linux64_mkl + - backports.zoneinfo=0.2.1=py310hff52083_8 + - billiard=3.6.4.0=py310h5764c6d_3 - blosc=1.21.5=h0f2a231_0 - - bottleneck=1.3.7=py311h1f0f07a_1 + - bottleneck=1.3.7=py310h1f7b6fc_1 - brotli=1.1.0=hd590300_1 - brotli-bin=1.1.0=hd590300_1 - - brotli-python=1.1.0=py311hb755f60_1 + - brotli-python=1.1.0=py310hc6cd4ac_1 - brunsli=0.1=h9c3ff4c_0 - bzip2=1.0.8=h7f98852_4 - c-ares=1.20.1=hd590300_0 @@ -40,7 +39,7 @@ dependencies: - cairo=1.18.0=h3faef2a_0 - celery=4.4.0=py_0 - certifi=2023.7.22=pyhd8ed1ab_0 - - cffi=1.16.0=py311hb3a22ac_0 + - cffi=1.16.0=py310h2fee648_0 - cfitsio=4.3.0=hbdc6101_0 - charls=2.4.2=h59595ed_0 - charset-normalizer=3.3.0=pyhd8ed1ab_0 @@ -48,11 +47,11 @@ dependencies: - click-plugins=1.1.1=py_0 - cligj=0.7.2=pyhd8ed1ab_1 - colorama=0.4.6=pyhd8ed1ab_0 - - contourpy=1.1.1=py311h9547e67_1 - - coverage=7.3.2=py311h459d7ec_0 - - cryptography=41.0.4=py311h63ff55d_0 + - contourpy=1.1.1=py310hd41b1e2_1 + - coverage=7.3.2=py310h2372a71_0 + - cryptography=41.0.4=py310h75e40e8_0 - cycler=0.12.1=pyhd8ed1ab_0 - - cython=0.29.36=py311hb755f60_1 + - cython=0.29.36=py310hc6cd4ac_1 - dav1d=1.2.1=hd590300_0 - decorator=5.1.1=pyhd8ed1ab_0 - dnspython=2.4.2=pyhd8ed1ab_0 @@ -61,7 +60,6 @@ dependencies: - exceptiongroup=1.1.3=pyhd8ed1ab_0 - executing=1.2.0=pyhd8ed1ab_0 - expat=2.5.0=hcb278e6_1 - - filelock=3.12.4=pyhd8ed1ab_0 - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 - font-ttf-inconsolata=3.000=h77eed37_0 - font-ttf-source-code-pro=2.038=h77eed37_0 @@ -69,36 +67,34 @@ dependencies: - fontconfig=2.14.2=h14ed4e7_0 - fonts-conda-ecosystem=1=0 - fonts-conda-forge=1=0 - - fonttools=4.43.1=py311h459d7ec_0 + - fonttools=4.43.1=py310h2372a71_0 - freetype=2.12.1=h267a509_2 - freexl=2.0.0=h743c826_0 - geos=3.12.0=h59595ed_0 - - geotiff=1.7.1=hee599c5_13 + - geotiff=1.7.1=hf074850_14 - gettext=0.21.1=h27087fc_0 - giflib=5.2.1=h0b41bf4_3 - - gmp=6.2.1=h58526e2_0 - - gmpy2=2.1.2=py311h6a5fa03_1 - - greenlet=3.0.0=py311hb755f60_1 + - greenlet=3.0.0=py310hc6cd4ac_1 - gwcs=0.19.0=pyhd8ed1ab_0 - h11=0.14.0=pyhd8ed1ab_0 - h2=4.1.0=pyhd8ed1ab_0 - - hdf4=4.2.15=h501b40f_6 + - hdf4=4.2.15=h2a13503_7 - hdf5=1.14.2=nompi_h4f84152_100 - hpack=4.0.0=pyh9f0ad1d_0 - httpcore=1.0.0=pyhd8ed1ab_0 - hyperframe=6.0.1=pyhd8ed1ab_0 - - hypothesis=6.87.3=pyha770c72_0 + - hypothesis=6.87.4=pyha770c72_0 - icu=73.2=h59595ed_0 - idna=3.4=pyhd8ed1ab_0 - - imagecodecs=2023.9.18=py311h9b38416_0 + - imagecodecs=2023.9.18=py310h496a806_2 - imageio=2.31.5=pyh8c1a49c_0 - importlib-metadata=6.8.0=pyha770c72_0 - importlib-resources=6.1.0=pyhd8ed1ab_0 - importlib_resources=6.1.0=pyhd8ed1ab_0 - iniconfig=2.0.0=pyhd8ed1ab_0 + - intel-openmp=2022.1.0=h9e868ea_3769 - ipython=8.16.1=pyh0d859eb_0 - jedi=0.19.1=pyhd8ed1ab_0 - - jinja2=3.1.2=pyhd8ed1ab_1 - jmespath=1.0.1=pyhd8ed1ab_0 - joblib=1.3.2=pyhd8ed1ab_0 - json-c=0.17=h7ab15ed_0 @@ -107,21 +103,21 @@ dependencies: - jxrlib=1.1=h7f98852_2 - kealib=1.5.2=hcd42e92_1 - keyutils=1.6.1=h166bdaf_0 - - kiwisolver=1.4.5=py311h9547e67_1 + - kiwisolver=1.4.5=py310hd41b1e2_1 - krb5=1.21.2=h659d440_0 - lazy_loader=0.3=pyhd8ed1ab_0 - - lcms2=2.15=h7f713cb_2 + - lcms2=2.15=hb7c19ff_3 - ld_impl_linux-64=2.40=h41732ed_0 - lerc=4.0.0=h27087fc_0 - libaec=1.1.2=h59595ed_1 - libarchive=3.7.2=h039dbb9_0 - libavif16=1.0.1=h87da1f6_2 - - libblas=3.9.0=16_linux64_mkl + - libblas=3.9.0=18_linux64_openblas - libboost-headers=1.82.0=ha770c72_6 - libbrotlicommon=1.1.0=hd590300_1 - libbrotlidec=1.1.0=hd590300_1 - libbrotlienc=1.1.0=hd590300_1 - - libcblas=3.9.0=16_linux64_mkl + - libcblas=3.9.0=18_linux64_openblas - libcurl=8.4.0=hca28451_0 - libdeflate=1.19=hd590300_0 - libedit=3.1.20191231=he28a2e2_2 @@ -129,28 +125,28 @@ dependencies: - libexpat=2.5.0=hcb278e6_1 - libffi=3.4.2=h7f98852_5 - libgcc-ng=13.2.0=h807b86a_2 - - libgdal=3.7.2=h3aa23ec_3 + - libgdal=3.7.2=h6f3d308_7 - libgfortran-ng=13.2.0=h69a702a_2 - libgfortran5=13.2.0=ha4646dd_2 - libglib=2.78.0=hebfc3b9_0 - - libhwloc=2.9.3=default_h554bfaf_1009 + - libgomp=13.2.0=h807b86a_2 - libiconv=1.17=h166bdaf_0 - - libjpeg-turbo=2.1.5.1=hd590300_1 + - libjpeg-turbo=3.0.0=hd590300_1 - libkml=1.3.0=h01aab08_1018 - - liblapack=3.9.0=16_linux64_mkl - - liblapacke=3.9.0=16_linux64_mkl + - liblapack=3.9.0=18_linux64_openblas - libnetcdf=4.9.2=nompi_h80fb2b6_112 - libnghttp2=1.52.0=h61bc06f_0 - libnsl=2.0.0=hd590300_1 + - libopenblas=0.3.24=pthreads_h413a1c8_0 - libpng=1.6.39=h753d276_0 - - libpq=15.4=hfc447b1_2 + - libpq=16.0=hfc447b1_1 - libprotobuf=3.20.3=h3eb15da_0 - librttopo=1.1.0=hb58d41b_14 - libspatialite=5.1.0=h090f1da_0 - libsqlite=3.43.2=h2797004_0 - libssh2=1.11.0=h0841786_0 - libstdcxx-ng=13.2.0=h7e041cc_2 - - libtiff=4.6.0=h29866fb_1 + - libtiff=4.6.0=ha9c0a0a_2 - libuuid=2.38.1=h0b41bf4_0 - libwebp-base=1.3.2=hd590300_0 - libxcb=1.15=h0b41bf4_0 @@ -158,57 +154,51 @@ dependencies: - libzip=1.10.1=h2629f0a_3 - libzlib=1.2.13=hd590300_5 - libzopfli=1.0.3=h9c3ff4c_0 - - llvm-openmp=15.0.7=h0cdce71_0 - lz4-c=1.9.4=hcb278e6_0 - lzo=2.10=h516909a_1000 - - markupsafe=2.1.3=py311h459d7ec_1 - - matplotlib-base=3.8.0=py311h54ef318_2 + - matplotlib-base=3.8.0=py310h62c0568_2 - matplotlib-inline=0.1.6=pyhd8ed1ab_0 - minizip=4.0.1=h0ab5242_5 - - mkl=2022.1.0=h84fe81f_915 - - mkl-devel=2022.1.0=ha770c72_916 - - mkl-include=2022.1.0=h84fe81f_915 + - mkl=2022.1.0=hc2b9512_224 - mock=5.1.0=pyhd8ed1ab_0 - - mpc=1.3.1=hfe3b2da_0 - - mpfr=4.2.0=hb012696_0 - - mpmath=1.3.0=pyhd8ed1ab_0 - munkres=1.1.4=pyh9f0ad1d_0 - mysql-common=8.0.33=hf1915f5_5 - - mysql-connector-python=8.0.32=py311h381d6c5_0 + - mysql-connector-python=8.0.32=py310h6eefaca_0 - mysql-libs=8.0.33=hca2cd23_5 - ncurses=6.4=hcb278e6_0 - networkx=3.1=pyhd8ed1ab_0 + - ninja=1.11.1=h924138e_0 - nspr=4.35=h27087fc_0 - nss=3.94=h1d7d5a4_0 - - numpy=1.23.5=py311h7d28db0_0 + - numpy=1.23.5=py310h53a5b5f_0 - openjpeg=2.5.0=h488ebb8_3 - openssl=3.1.3=hd590300_0 - packaging=23.2=pyhd8ed1ab_0 - parso=0.8.3=pyhd8ed1ab_0 - pcre2=10.40=hc3806b6_0 - pexpect=4.8.0=pyh1a96a4e_2 - - photutils=1.9.0=py311h459d7ec_0 + - photutils=1.9.0=py310h2372a71_0 - pickleshare=0.7.5=py_1003 - - pillow=10.0.1=py311h8aef010_1 + - pillow=10.0.1=py310h01dd4db_2 - pip=23.2.1=pyhd8ed1ab_0 - pixman=0.42.2=h59595ed_0 - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1 - pluggy=1.3.0=pyhd8ed1ab_0 - - poppler=23.08.0=hf2349cb_2 + - poppler=23.10.0=h590f24d_0 - poppler-data=0.4.12=hd8ed1ab_0 - - postgresql=15.4=h8972f4a_2 + - postgresql=16.0=h8972f4a_1 - proj=9.3.0=h1d62c97_1 - prompt-toolkit=3.0.39=pyha770c72_0 - prompt_toolkit=3.0.39=hd8ed1ab_0 - - protobuf=3.20.3=py311hcafe171_1 - - psutil=5.9.5=py311h459d7ec_1 - - psycopg2=2.9.7=py311h68d4568_0 - - psycopg2-binary=2.9.7=pyhd8ed1ab_0 + - protobuf=3.20.3=py310heca2aa9_1 + - psutil=5.9.5=py310h2372a71_1 + - psycopg2=2.9.7=py310h275853b_1 + - psycopg2-binary=2.9.7=pyhd8ed1ab_1 - pthread-stubs=0.4=h36c2ea0_1001 - ptyprocess=0.7.0=pyhd3deb0d_0 - pure_eval=0.2.2=pyhd8ed1ab_0 - pycparser=2.21=pyhd8ed1ab_0 - - pyerfa=2.0.0.3=py311h1f0f07a_1 + - pyerfa=2.0.0.3=py310h1f7b6fc_1 - pygments=2.16.1=pyhd8ed1ab_0 - pyparsing=3.1.1=pyhd8ed1ab_0 - pysocks=1.7.1=pyha2e5f31_6 @@ -222,39 +212,37 @@ dependencies: - pytest-mock=3.11.1=pyhd8ed1ab_0 - pytest-openfiles=0.5.0=py_0 - pytest-remotedata=0.4.1=pyhd8ed1ab_0 - - python=3.11.6=hab00c5b_0_cpython + - python=3.10.12=hd12c33a_0_cpython - python-dateutil=2.8.2=pyhd8ed1ab_0 - - python_abi=3.11=4_cp311 - - pytorch=2.1.0=py3.11_cpu_0 - - pytorch-mutex=1.0=cpu + - python_abi=3.10=4_cp310 + - pytorch=1.12.1=cpu_py310h75c9ab6_0 - pytz=2023.3.post1=pyhd8ed1ab_0 - - pywavelets=1.4.1=py311h1f0f07a_1 - - pyyaml=6.0.1=py311h459d7ec_1 - - rasterio=1.3.8=py311h40fbdff_3 + - pywavelets=1.4.1=py310h1f7b6fc_1 + - pyyaml=6.0.1=py310h2372a71_1 + - rasterio=1.3.8=py310h6a913dc_4 - rav1e=0.6.6=he8a937b_2 - readline=8.2=h8228510_1 - referencing=0.30.2=pyhd8ed1ab_0 - requests=2.31.0=pyhd8ed1ab_0 - - rpds-py=0.10.4=py311h46250e7_0 - - scikit-image=0.22.0=py311h320fe9a_2 - - scikit-learn=1.3.1=py311hc009520_1 - - scipy=1.11.3=py311h64a7726_1 + - rpds-py=0.10.6=py310hcb5633a_0 + - scikit-image=0.22.0=py310hcc13569_2 + - scikit-learn=1.3.1=py310h1fdf081_1 + - scipy=1.11.3=py310hb13e2d6_1 - semantic_version=2.10.0=pyhd8ed1ab_0 - - sep=1.2.1=py311h1f0f07a_2 + - sep=1.2.1=py310h1f7b6fc_2 - setuptools=68.2.2=pyhd8ed1ab_0 - - shapely=2.0.1=py311he06c224_3 - - simplejson=3.19.2=py311h459d7ec_0 + - shapely=2.0.2=py310h7dcad9a_0 + - simplejson=3.19.2=py310h2372a71_0 - six=1.16.0=pyh6c4a22f_0 + - sleef=3.5.1=h9b69904_2 - snappy=1.1.10=h9fff704_0 - sniffio=1.3.0=pyhd8ed1ab_0 - snuggs=1.4.7=py_0 - sortedcontainers=2.4.0=pyhd8ed1ab_0 - - sqlalchemy=2.0.21=py311h459d7ec_1 + - sqlalchemy=2.0.22=py310h2372a71_0 - sqlite=3.43.2=h2c6b66d_0 - stack_data=0.6.2=pyhd8ed1ab_0 - svt-av1=1.7.0=h59595ed_0 - - sympy=1.12=pypyh9d50eac_103 - - tbb=2021.10.0=h00ab1b0_1 - tenacity=8.2.3=pyhd8ed1ab_0 - threadpoolctl=3.2.0=pyha21a80b_0 - tifffile=2023.9.26=pyhd8ed1ab_0 @@ -268,7 +256,8 @@ dependencies: - typing_extensions=4.8.0=pyha770c72_0 - tzcode=2023c=h0b41bf4_0 - tzdata=2023c=h71feb2d_0 - - tzlocal=5.1=py311h38be061_0 + - tzlocal=5.1=py310hff52083_0 + - unicodedata2=15.1.0=py310h2372a71_0 - uriparser=0.9.7=hcb278e6_1 - wcwidth=0.2.8=pyhd8ed1ab_0 - wheel=0.41.2=pyhd8ed1ab_0 @@ -276,7 +265,7 @@ dependencies: - xorg-kbproto=1.0.7=h7f98852_1002 - xorg-libice=1.1.1=hd590300_0 - xorg-libsm=1.2.4=h7391055_0 - - xorg-libx11=1.8.6=h8ee46fc_0 + - xorg-libx11=1.8.7=h8ee46fc_0 - xorg-libxau=1.0.11=hd590300_0 - xorg-libxdmcp=1.1.3=h7f98852_0 - xorg-libxext=1.3.4=h0b41bf4_2 @@ -294,8 +283,9 @@ dependencies: - pip: - asciitree==0.3.3 - astropy-healpix==1.0.0 - - boto3==1.28.62 - - botocore==1.31.62 + - async-timeout==4.0.3 + - boto3==1.28.63 + - botocore==1.31.63 - cloudpickle==2.2.1 - cosmic-conn==0.4.1 - dask==2023.9.3 @@ -320,4 +310,3 @@ dependencies: - urllib3==1.26.17 - vine==1.3.0 - zarr==2.16.1 -prefix: /root/micromamba/envs/banzai From cf614fd8675d3991b4be6b155ac5b72ad37faed3 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Fri, 20 Oct 2023 09:45:06 -0600 Subject: [PATCH 07/13] Fixes to photometry stages. --- banzai/photometry.py | 4 ++-- banzai/settings.py | 3 --- banzai/tests/e2e-k8s.yaml | 4 ++-- environment.yaml | 2 ++ 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/banzai/photometry.py b/banzai/photometry.py index b803a1e5..01b933f7 100755 --- a/banzai/photometry.py +++ b/banzai/photometry.py @@ -109,7 +109,7 @@ def do_stage(self, image): deblended_seg_map = deblend_sources(convolved_data, segmentation_map, npixels=self.min_area, nlevels=32, contrast=0.005, progress_bar=False, - nproc=self.runtime_context.N_PHOT_CORES) + nproc=1) # Convert the segmentation map to a source catalog catalog = SourceCatalog(data, deblended_seg_map, convolved_data=convolved_data, error=error, background=bkg.background) @@ -132,7 +132,7 @@ def do_stage(self, image): for r in [0.25, 0.5, 0.75]: sources['fluxrad' + f'{r:.2f}'.lstrip("0.")] = catalog.fluxfrac_radius(r) - + sources['flag'] = 0 # Flag = 1 for sources with bad pixels diff --git a/banzai/settings.py b/banzai/settings.py index f674657e..d85197b1 100644 --- a/banzai/settings.py +++ b/banzai/settings.py @@ -156,6 +156,3 @@ CELERY_TASK_QUEUE_NAME = os.getenv('CELERY_TASK_QUEUE_NAME', 'celery') REFERENCE_CATALOG_URL = os.getenv('REFERENCE_CATALOG_URL', 'http://phot-catalog.lco.gtn/') - -# Number of cores to use for photometry deblending -N_PHOT_CORES = 4 diff --git a/banzai/tests/e2e-k8s.yaml b/banzai/tests/e2e-k8s.yaml index 02d9ea7d..448fb9fb 100644 --- a/banzai/tests/e2e-k8s.yaml +++ b/banzai/tests/e2e-k8s.yaml @@ -118,8 +118,8 @@ spec: - "banzai-celery-worker" - --concurrency - "4" - - -l - - "debug" + - "-l" + - "info" - "-Q" - "$(CELERY_TASK_QUEUE_NAME)" readinessProbe: diff --git a/environment.yaml b/environment.yaml index 76094ea8..e01b474c 100644 --- a/environment.yaml +++ b/environment.yaml @@ -1,3 +1,5 @@ +# This environment was produced with the following: +# conda create -n banzai python=3.10 'cython<3' 'numpy<1.24' bottleneck scipy astropy pytest mock requests ipython coverage pyyaml kombu sep 'elasticsearch<6.0.0,>=5.0.0' pytest-astropy mysql-connector-python photutils psycopg2-binary tenacity 'amqp<3' 'celery[redis]>=4.3.1,<5' scikit-image emcee python-dateutil 'sqlalchemy>=1.3.0b1' psycopg2-binary apscheduler 'pytorch>=1.6.0' --channel conda-forge --channel astropy --channel pytorch --solver=libmamba channels: - pytorch - conda-forge From 367b0ba5ea34dc729966cf9c88664663b78c07d5 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Fri, 20 Oct 2023 14:32:00 -0600 Subject: [PATCH 08/13] Fixes to photometry stage. --- banzai/photometry.py | 16 +++++++--------- banzai/tests/e2e-k8s.yaml | 6 +++--- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/banzai/photometry.py b/banzai/photometry.py index 01b933f7..fcb114e0 100755 --- a/banzai/photometry.py +++ b/banzai/photometry.py @@ -30,17 +30,17 @@ def radius_of_contour(contour, source): def flag_sources(sources, source_labels, segmentation_map, mask, flag, mask_value): - affected_sources = np.unique(segmentation_map[mask == mask_value]) + affected_sources = np.unique(segmentation_map.data[mask == mask_value]) sources['flag'][np.in1d(source_labels, affected_sources)] |= flag def flag_deblended(sources, catalog, segmentation_map, deblended_seg_map, flag_value=2): # By default deblending appends labels instead of reassigning them so we can just use the # extras in the deblended map - deblended_sources = np.unique(deblended_seg_map[deblended_seg_map > np.max(segmentation_map)]) + deblended_sources = np.unique(deblended_seg_map.data[deblended_seg_map > np.max(segmentation_map)]) # Get the sources that were originally blended - original_blends = np.unique(segmentation_map[deblended_seg_map > np.max(segmentation_map)]) - deblended_sources = np.hstack([deblend_sources, original_blends]) + original_blends = np.unique(segmentation_map.data[deblended_seg_map > np.max(segmentation_map)]) + deblended_sources = np.hstack([deblended_sources, original_blends]) sources['flag'][np.in1d(catalog.labels, deblended_sources)] |= flag_value @@ -76,9 +76,6 @@ def __init__(self, runtime_context): def do_stage(self, image): try: - # Increase the internal buffer size in sep. This is most necessary for crowded fields. - ny, nx = image.shape - data = image.data.copy() error = image.uncertainty @@ -101,10 +98,11 @@ def do_stage(self, image): kernel_squared = CustomKernel(kernel.array * kernel.array) normalization = np.sqrt(convolve(1 / (error * error), kernel_squared)) convolved_data /= normalization - + logger.info('Running image segmentation', image=image) # Do an initial source detection segmentation_map = detect_sources(convolved_data, self.threshold, npixels=self.min_area) + logger.info('Deblending sources', image=image) # Note that nlevels here is DEBLEND_NTHRESH in source extractor which is 32 by default deblended_seg_map = deblend_sources(convolved_data, segmentation_map, npixels=self.min_area, nlevels=32, @@ -255,7 +253,7 @@ def do_stage(self, image): image.meta['L1ELLIP'] = (mean_ellipticity, 'Mean image ellipticity (1-B/A)') mean_position_angle = stats.sigma_clipped_mean(catalog['theta'][good_objects], 3.0) - image.meta['L1ELLIPA'] = (mean_position_angle,'[deg] PA of mean image ellipticity') + image.meta['L1ELLIPA'] = (mean_position_angle, '[deg] PA of mean image ellipticity') logging_tags = {key: float(image.meta[key]) for key in ['L1MEAN', 'L1MEDIAN', 'L1SIGMA', 'L1FWHM', 'L1ELLIP', 'L1ELLIPA']} diff --git a/banzai/tests/e2e-k8s.yaml b/banzai/tests/e2e-k8s.yaml index 448fb9fb..fbdb2172 100644 --- a/banzai/tests/e2e-k8s.yaml +++ b/banzai/tests/e2e-k8s.yaml @@ -94,13 +94,13 @@ spec: - name: TASK_HOST value: "redis://localhost:6379/0" - name: BANZAI_WORKER_LOGLEVEL - value: debug + value: info - name: CALIBRATE_PROPOSAL_ID value: "calibrate" - name: OBSERVATION_PORTAL_URL value: "http://internal-observation-portal.lco.gtn/api/observations/" - name: OMP_NUM_THREADS - value: "4" + value: "2" - name: FITS_EXCHANGE value: "fits_files" - name: OPENTSDB_PYTHON_METRICS_TEST_MODE @@ -117,7 +117,7 @@ spec: - --hostname - "banzai-celery-worker" - --concurrency - - "4" + - "2" - "-l" - "info" - "-Q" From a5d24ba4eaa81a47d86ae99b314195aa6c64f8aa Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Sat, 21 Oct 2023 12:15:17 -0600 Subject: [PATCH 09/13] More fixes to photometry stage --- banzai/photometry.py | 142 +++++++++++++++++++------------------- banzai/tests/e2e-k8s.yaml | 26 ------- 2 files changed, 71 insertions(+), 97 deletions(-) diff --git a/banzai/photometry.py b/banzai/photometry.py index fcb114e0..0823c129 100755 --- a/banzai/photometry.py +++ b/banzai/photometry.py @@ -23,8 +23,8 @@ def radius_of_contour(contour, source): x = contour[:, 1] y = contour[:, 0] - x_center = (source['xmax'] - source['xmin'] + 1) / 2.0 - 0.5 - y_center = (source['ymax'] - source['ymin'] + 1) / 2.0 - 0.5 + x_center = (source.bbox_xmax - source.bbox_xmin + 1) / 2.0 - 0.5 + y_center = (source.bbox_ymax - source.bbox_ymin + 1) / 2.0 - 0.5 return np.percentile(np.sqrt((x - x_center)**2.0 + (y - y_center) ** 2.0), 90) @@ -119,7 +119,7 @@ def do_stage(self, image): 'a': catalog.semimajor_sigma.value, 'b': catalog.semiminor_sigma.value, 'theta': catalog.orientation.to('deg').value, 'ellipticity': catalog.ellipticity.value, 'kronrad': catalog.kron_radius.value, - 'flux': catalog.kron_flux, 'fluxerr': catalog, + 'flux': catalog.kron_flux, 'fluxerr': catalog.kron_fluxerr, 'x2': catalog.covar_sigx2.value, 'y2': catalog.covar_sigy2.value, 'xy': catalog.covar_sigxy.value, 'background': catalog.background_mean}) @@ -152,91 +152,91 @@ def do_stage(self, image): sources['fwhm'] = np.nan sources['fwtm'] = np.nan # Here we estimate contours - for source in sources: + for source, row in zip(sources, catalog): if source['flag'] == 0: for ratio, keyword in zip([0.5, 0.1], ['fwhm', 'fwtm']): - contours = measure.find_contours(data[source['ymin']: source['ymax'] + 1, - source['xmin']: source['xmax'] + 1], + contours = measure.find_contours(data[row.bbox_ymin: row.bbox_ymax + 1, + row.bbox_xmin: row.bbox_xmax + 1], ratio * source['peak']) if contours: # If there are multiple contours like a donut might have take the outer - contour_radii = [radius_of_contour(contour, source) for contour in contours] + contour_radii = [radius_of_contour(contour, row) for contour in contours] source[keyword] = 2.0 * np.nanmax(contour_radii) # Add the units and description to the catalogs - catalog['x'].unit = 'pixel' - catalog['x'].description = 'X coordinate of the object' - catalog['y'].unit = 'pixel' - catalog['y'].description = 'Y coordinate of the object' - catalog['xwin'].unit = 'pixel' - catalog['xwin'].description = 'Windowed X coordinate of the object' - catalog['ywin'].unit = 'pixel' - catalog['ywin'].description = 'Windowed Y coordinate of the object' - catalog['xpeak'].unit = 'pixel' - catalog['xpeak'].description = 'X coordinate of the peak' - catalog['ypeak'].unit = 'pixel' - catalog['ypeak'].description = 'Windowed Y coordinate of the peak' - catalog['flux'].unit = 'count' - catalog['flux'].description = 'Flux within a Kron-like elliptical aperture' - catalog['fluxerr'].unit = 'count' - catalog['fluxerr'].description = 'Error on the flux within Kron aperture' - catalog['peak'].unit = 'count' - catalog['peak'].description = 'Peak flux (flux at xpeak, ypeak)' + sources['x'].unit = 'pixel' + sources['x'].description = 'X coordinate of the object' + sources['y'].unit = 'pixel' + sources['y'].description = 'Y coordinate of the object' + sources['xwin'].unit = 'pixel' + sources['xwin'].description = 'Windowed X coordinate of the object' + sources['ywin'].unit = 'pixel' + sources['ywin'].description = 'Windowed Y coordinate of the object' + sources['xpeak'].unit = 'pixel' + sources['xpeak'].description = 'X coordinate of the peak' + sources['ypeak'].unit = 'pixel' + sources['ypeak'].description = 'Windowed Y coordinate of the peak' + sources['flux'].unit = 'count' + sources['flux'].description = 'Flux within a Kron-like elliptical aperture' + sources['fluxerr'].unit = 'count' + sources['fluxerr'].description = 'Error on the flux within Kron aperture' + sources['peak'].unit = 'count' + sources['peak'].description = 'Peak flux (flux at xpeak, ypeak)' for diameter in [1, 2, 3, 4, 5, 6]: - catalog['fluxaper{0}'.format(diameter)].unit = 'count' - catalog['fluxaper{0}'.format(diameter)].description = 'Flux from fixed circular aperture: {0}" diameter'.format(diameter) - catalog['fluxerr{0}'.format(diameter)].unit = 'count' - catalog['fluxerr{0}'.format(diameter)].description = 'Error on Flux from circular aperture: {0}"'.format(diameter) - - catalog['background'].unit = 'count' - catalog['background'].description = 'Average background value in the aperture' - catalog['fwhm'].unit = 'pixel' - catalog['fwhm'].description = 'FWHM of the object' - catalog['fwtm'].unit = 'pixel' - catalog['fwtm'].description = 'Full-Width Tenth Maximum' - catalog['a'].unit = 'pixel' - catalog['a'].description = 'Semi-major axis of the object' - catalog['b'].unit = 'pixel' - catalog['b'].description = 'Semi-minor axis of the object' - catalog['theta'].unit = 'degree' - catalog['theta'].description = 'Position angle of the object' - catalog['kronrad'].unit = 'pixel' - catalog['kronrad'].description = 'Kron radius used for extraction' - catalog['ellipticity'].description = 'Ellipticity' - catalog['fluxrad25'].unit = 'pixel' - catalog['fluxrad25'].description = 'Radius containing 25% of the flux' - catalog['fluxrad50'].unit = 'pixel' - catalog['fluxrad50'].description = 'Radius containing 50% of the flux' - catalog['fluxrad75'].unit = 'pixel' - catalog['fluxrad75'].description = 'Radius containing 75% of the flux' - catalog['x2'].unit = 'pixel^2' - catalog['x2'].description = 'Variance on X coordinate of the object' - catalog['y2'].unit = 'pixel^2' - catalog['y2'].description = 'Variance on Y coordinate of the object' - catalog['xy'].unit = 'pixel^2' - catalog['xy'].description = 'XY covariance of the object' - catalog['flag'].description = 'Bit mask of extraction/photometry flags' - - catalog.sort('flux') - catalog.reverse() + sources['fluxaper{0}'.format(diameter)].unit = 'count' + sources['fluxaper{0}'.format(diameter)].description = 'Flux from fixed circular aperture: {0}" diameter'.format(diameter) + sources['fluxerr{0}'.format(diameter)].unit = 'count' + sources['fluxerr{0}'.format(diameter)].description = 'Error on Flux from circular aperture: {0}"'.format(diameter) + + sources['background'].unit = 'count' + sources['background'].description = 'Average background value in the aperture' + sources['fwhm'].unit = 'pixel' + sources['fwhm'].description = 'FWHM of the object' + sources['fwtm'].unit = 'pixel' + sources['fwtm'].description = 'Full-Width Tenth Maximum' + sources['a'].unit = 'pixel' + sources['a'].description = 'Semi-major axis of the object' + sources['b'].unit = 'pixel' + sources['b'].description = 'Semi-minor axis of the object' + sources['theta'].unit = 'degree' + sources['theta'].description = 'Position angle of the object' + sources['kronrad'].unit = 'pixel' + sources['kronrad'].description = 'Kron radius used for extraction' + sources['ellipticity'].description = 'Ellipticity' + sources['fluxrad25'].unit = 'pixel' + sources['fluxrad25'].description = 'Radius containing 25% of the flux' + sources['fluxrad50'].unit = 'pixel' + sources['fluxrad50'].description = 'Radius containing 50% of the flux' + sources['fluxrad75'].unit = 'pixel' + sources['fluxrad75'].description = 'Radius containing 75% of the flux' + sources['x2'].unit = 'pixel^2' + sources['x2'].description = 'Variance on X coordinate of the object' + sources['y2'].unit = 'pixel^2' + sources['y2'].description = 'Variance on Y coordinate of the object' + sources['xy'].unit = 'pixel^2' + sources['xy'].description = 'XY covariance of the object' + sources['flag'].description = 'Bit mask of extraction/photometry flags' + + sources.sort('flux') + sources.reverse() # Save some background statistics in the header - mean_background = stats.sigma_clipped_mean(bkg.background(), 5.0) + mean_background = stats.sigma_clipped_mean(bkg.background, 5.0) image.meta['L1MEAN'] = (mean_background, '[counts] Sigma clipped mean of frame background') - median_background = np.median(bkg.background()) + median_background = np.median(bkg.background) image.meta['L1MEDIAN'] = (median_background, '[counts] Median of frame background') - std_background = stats.robust_standard_deviation(bkg.background()) + std_background = stats.robust_standard_deviation(bkg.background) image.meta['L1SIGMA'] = (std_background, '[counts] Robust std dev of frame background') # Save some image statistics to the header - good_objects = catalog['flag'] == 0 + good_objects = sources['flag'] == 0 for quantity in ['fwhm', 'ellipticity', 'theta']: - good_objects = np.logical_and(good_objects, np.logical_not(np.isnan(catalog[quantity]))) + good_objects = np.logical_and(good_objects, np.logical_not(np.isnan(sources[quantity]))) if good_objects.sum() == 0: image.meta['L1FWHM'] = ('NaN', '[arcsec] Frame FWHM in arcsec') image.meta['L1FWTM'] = ('NaN', 'Ratio of FWHM to Full-Width Tenth Max') @@ -244,15 +244,15 @@ def do_stage(self, image): image.meta['L1ELLIP'] = ('NaN', 'Mean image ellipticity (1-B/A)') image.meta['L1ELLIPA'] = ('NaN', '[deg] PA of mean image ellipticity') else: - seeing = np.nanmedian(catalog['fwhm'][good_objects]) * image.pixel_scale + seeing = np.nanmedian(sources['fwhm'][good_objects]) * image.pixel_scale image.meta['L1FWHM'] = (seeing, '[arcsec] Frame FWHM in arcsec') - image.meta['L1FWTM'] = (np.nanmedian(catalog['fwtm'][good_objects] / catalog['fwhm'][good_objects]), + image.meta['L1FWTM'] = (np.nanmedian(sources['fwtm'][good_objects] / sources['fwhm'][good_objects]), 'Ratio of FWHM to Full-Width Tenth Max') - mean_ellipticity = stats.sigma_clipped_mean(catalog['ellipticity'][good_objects], 3.0) + mean_ellipticity = stats.sigma_clipped_mean(sources['ellipticity'][good_objects], 3.0) image.meta['L1ELLIP'] = (mean_ellipticity, 'Mean image ellipticity (1-B/A)') - mean_position_angle = stats.sigma_clipped_mean(catalog['theta'][good_objects], 3.0) + mean_position_angle = stats.sigma_clipped_mean(sources['theta'][good_objects], 3.0) image.meta['L1ELLIPA'] = (mean_position_angle, '[deg] PA of mean image ellipticity') logging_tags = {key: float(image.meta[key]) for key in ['L1MEAN', 'L1MEDIAN', 'L1SIGMA', @@ -260,7 +260,7 @@ def do_stage(self, image): logger.info('Extracted sources', image=image, extra_tags=logging_tags) # adding catalog (a data table) to the appropriate images attribute. - image.add_or_update(DataTable(catalog, name='CAT')) + image.add_or_update(DataTable(sources, name='CAT')) except Exception: logger.error(logs.format_exception(), image=image) return image diff --git a/banzai/tests/e2e-k8s.yaml b/banzai/tests/e2e-k8s.yaml index fbdb2172..7c3f9f0f 100644 --- a/banzai/tests/e2e-k8s.yaml +++ b/banzai/tests/e2e-k8s.yaml @@ -142,32 +142,6 @@ spec: limits: cpu: 8 memory: 8Gi - - name: banzai-celery-beat - image: @BANZAI_IMAGE@ - imagePullPolicy: IfNotPresent - volumeMounts: - - name: banzai-data - mountPath: /archive/engineering - subPath: engineering - readOnly: false - env: - - name: DB_ADDRESS - value: "sqlite:////archive/engineering/test.db" - - name: RETRY_DELAY - value: "0" - - name: TASK_HOST - value: "redis://localhost:6379/0" - - name: CELERY_TASK_QUEUE_NAME - value: "e2e_task_queue" - command: - - banzai_automate_stack_calibrations - resources: - requests: - cpu: 0.1 - memory: 1Gi - limits: - cpu: 1 - memory: 1Gi - name: banzai-listener image: @BANZAI_IMAGE@ imagePullPolicy: IfNotPresent From 18fef45a184bc306800843e582967f9b2af16a51 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Mon, 30 Oct 2023 13:39:11 -0400 Subject: [PATCH 10/13] Fixes to actually call flagging sources that fall off the edge --- banzai/photometry.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/banzai/photometry.py b/banzai/photometry.py index 0823c129..d395c050 100755 --- a/banzai/photometry.py +++ b/banzai/photometry.py @@ -64,7 +64,7 @@ def flag_edge_sources(image, sources, flag_value=8): sources_off = np.logical_or(sources_off, major_xmax > nx) sources_off = np.logical_or(sources_off, minor_ymax > ny) sources_off = np.logical_or(sources_off, major_ymax > ny) - sources[sources_off] |= flag_value + sources[sources_off]['flag'] |= flag_value class SourceDetector(Stage): @@ -107,7 +107,8 @@ def do_stage(self, image): deblended_seg_map = deblend_sources(convolved_data, segmentation_map, npixels=self.min_area, nlevels=32, contrast=0.005, progress_bar=False, - nproc=1) + nproc=1, mode='sinh') + logger.info('Finished deblending. Estimat', image=image) # Convert the segmentation map to a source catalog catalog = SourceCatalog(data, deblended_seg_map, convolved_data=convolved_data, error=error, background=bkg.background) @@ -140,6 +141,7 @@ def do_stage(self, image): # Flag = 4 for sources that have saturated pixels flag_sources(sources, catalog.labels, deblended_seg_map, image.mask, flag=4, mask_value=2) # Flag = 8 if kron aperture falls off the image + flag_edge_sources(image, sources, flag_pixel=8) # Flag = 16 if source has cosmic ray pixels flag_sources(sources, catalog.labels, deblended_seg_map, image.mask, flag=16, mask_value=8) From 53500572f0f793388753d8ea0cf19053c3cc2301 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Wed, 14 Feb 2024 12:18:25 -0500 Subject: [PATCH 11/13] Include file caching for the e2e tests. --- helm-chart/banzai/templates/listener.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/helm-chart/banzai/templates/listener.yaml b/helm-chart/banzai/templates/listener.yaml index d4afa66b..9d3429c9 100644 --- a/helm-chart/banzai/templates/listener.yaml +++ b/helm-chart/banzai/templates/listener.yaml @@ -98,7 +98,6 @@ spec: - "--db-address=$(DB_ADDRESS)" - "--broker-url=$(FITS_BROKER)" - "--queue-name=$(QUEUE_NAME)" - - "--no-file-cache" env: {{- include "banzai.Env" . | nindent 12 }} resources: @@ -128,7 +127,6 @@ spec: - "--db-address=$(DB_ADDRESS)" - "--broker-url=$(FITS_BROKER)" - "--log-level=info" - - "--no-file-cache" env: {{- include "banzai.Env" . | nindent 12 }} resources: From 47b97729c32e8eee886d4eaabe49719351fc8c27 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Wed, 21 Feb 2024 15:19:16 -0500 Subject: [PATCH 12/13] Fixes to e2e tests. --- .github/workflows/e2e.yaml | 2 +- MANIFEST.in | 1 + banzai/photometry.py | 4 ++-- banzai/settings.py | 2 +- banzai/tests/test_end_to_end.py | 8 ++++---- pytest.ini | 12 ++++++------ 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index e71aa7a2..d43efe84 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -45,7 +45,7 @@ jobs: - name: Test Master Bias Creation run: | - kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias - name: Cleanup run: | diff --git a/MANIFEST.in b/MANIFEST.in index 390ebf70..37d47f2f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -13,3 +13,4 @@ prune docs/_build prune docs/api include pyproject.toml +include pytest.ini diff --git a/banzai/photometry.py b/banzai/photometry.py index d395c050..e48fce32 100755 --- a/banzai/photometry.py +++ b/banzai/photometry.py @@ -108,7 +108,7 @@ def do_stage(self, image): npixels=self.min_area, nlevels=32, contrast=0.005, progress_bar=False, nproc=1, mode='sinh') - logger.info('Finished deblending. Estimat', image=image) + logger.info('Finished deblending. Saving catalog', image=image) # Convert the segmentation map to a source catalog catalog = SourceCatalog(data, deblended_seg_map, convolved_data=convolved_data, error=error, background=bkg.background) @@ -141,7 +141,7 @@ def do_stage(self, image): # Flag = 4 for sources that have saturated pixels flag_sources(sources, catalog.labels, deblended_seg_map, image.mask, flag=4, mask_value=2) # Flag = 8 if kron aperture falls off the image - flag_edge_sources(image, sources, flag_pixel=8) + flag_edge_sources(image, sources, flag_value=8) # Flag = 16 if source has cosmic ray pixels flag_sources(sources, catalog.labels, deblended_seg_map, image.mask, flag=16, mask_value=8) diff --git a/banzai/settings.py b/banzai/settings.py index 63953e30..fdfb98b5 100644 --- a/banzai/settings.py +++ b/banzai/settings.py @@ -80,7 +80,7 @@ 'elp': {'minute': 0, 'hour': 23}, 'ogg': {'minute': 0, 'hour': 3}} -ASTROMETRY_SERVICE_URL = os.getenv('ASTROMETRY_SERVICE_URL', 'http://astrometry.lco.gtn/catalog/') +ASTROMETRY_SERVICE_URL = os.getenv('ASTROMETRY_SERVICE_URL', ' ') CALIBRATION_FILENAME_FUNCTIONS = {'BIAS': ('banzai.utils.file_utils.config_to_filename', 'banzai.utils.file_utils.ccdsum_to_filename'), diff --git a/banzai/tests/test_end_to_end.py b/banzai/tests/test_end_to_end.py index f748aeb0..b879ca69 100644 --- a/banzai/tests/test_end_to_end.py +++ b/banzai/tests/test_end_to_end.py @@ -124,7 +124,7 @@ def get_expected_number_of_calibrations(raw_filename_pattern, calibration_type): if site in frame['filename'] and instrument in frame['filename'] and dayobs in frame['filename'] and raw_filename_pattern in frame['filename'] ] - if 'calibration_type.lower()' == 'skyflat': + if calibration_type.lower() == 'skyflat': # Group by filter observed_filters = [] for frame in raw_frames_for_this_dayobs: @@ -244,7 +244,7 @@ def test_if_stacked_flat_frame_was_created(self): @pytest.mark.e2e @pytest.mark.science_files class TestScienceFileCreation: - @pytest.fixture(autouse=True) + @pytest.fixture(autouse=True, scope='class') @mock.patch('banzai.utils.observation_utils.requests.get', side_effect=observation_portal_side_effect) def reduce_science_frames(self, mock_observation_portal): run_reduce_individual_frames('e00.fits') @@ -254,7 +254,7 @@ def test_if_science_frames_were_created(self): created_files = [] for day_obs in DAYS_OBS: expected_files += [filename.replace('e00', 'e91') - for filename in TEST_FRAMES['filename']] + for filename in TEST_FRAMES['filename'] if 'e00.fits' in filename] created_files += [os.path.basename(filename) for filename in glob(os.path.join(DATA_ROOT, day_obs, 'processed', '*e91*'))] assert len(expected_files) > 0 @@ -266,6 +266,6 @@ def test_that_photometric_calibration_succeeded(self): for day_obs in DAYS_OBS: science_files += [filepath for filepath in glob(os.path.join(DATA_ROOT, day_obs, 'processed', '*e91*'))] - zeropoints = [fits.open(file)['SCI'].header.get('L1ZP') for file in science_files] + zeropoints = [fits.open(filename)['SCI'].header.get('L1ZP') for filename in science_files] # check that at least one of our images contains a zeropoint assert zeropoints.count(None) != len(zeropoints) diff --git a/pytest.ini b/pytest.ini index b30fc1bd..99a221fe 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,11 +1,11 @@ [pytest] -minversion = 3.5 -norecursedirs = build docs/_build -doctest_plus = enabled -addopts = -p no:warnings +minversion = "3.5" +norecursedirs = "build docs/_build" +doctest_plus = "enabled" +addopts = "-p no:warnings" log_cli = True -log_level = info -log_cli_level = info +log_level = "info" +log_cli_level = "info" markers = # E2E test markers e2e : End-to-end test suite From 39dcf4b524a1e524b2176c6118e905fd4d2892d8 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Wed, 21 Feb 2024 15:40:17 -0500 Subject: [PATCH 13/13] Fixes to pytest.ini format --- pytest.ini | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pytest.ini b/pytest.ini index 99a221fe..b30fc1bd 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,11 +1,11 @@ [pytest] -minversion = "3.5" -norecursedirs = "build docs/_build" -doctest_plus = "enabled" -addopts = "-p no:warnings" +minversion = 3.5 +norecursedirs = build docs/_build +doctest_plus = enabled +addopts = -p no:warnings log_cli = True -log_level = "info" -log_cli_level = "info" +log_level = info +log_cli_level = info markers = # E2E test markers e2e : End-to-end test suite