diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml new file mode 100644 index 00000000..0ae74e62 --- /dev/null +++ b/.github/workflows/docker_image.yml @@ -0,0 +1,53 @@ +name: Docker Image + +concurrency: ${{ github.workflow }}-${{ github.ref }} + +on: + push: + tags: + - "*" + +jobs: + build-tag: + + runs-on: ubuntu-latest + + permissions: + contents: write + packages: write + + env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + + steps: + - name: "Checkout" + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + install: true + + - name: Login to Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and also push Dockerimage + id: build-and-push + uses: docker/build-push-action@v3 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/build.yml b/.github/workflows/unit-tests.yml similarity index 80% rename from .github/workflows/build.yml rename to .github/workflows/unit-tests.yml index f7608ec3..3c8e4e77 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/unit-tests.yml @@ -12,14 +12,10 @@ jobs: fail-fast: false matrix: include: - - name: Python 3.7 with required dependencies - os: macos-latest - python-version: 3.7 - toxenv: py37-test - name: Documentation build os: ubuntu-latest - python-version: 3.8 + python-version: 3.9 toxenv: build_docs - name: Python 3.9 with developer version of astropy and numpy @@ -27,16 +23,16 @@ jobs: python-version: 3.9 toxenv: py39-test-devdeps - - name: Python 3.7 with older dependencies, astropy 3.0 and Numpy 1.17 + - name: Python 3.8 with minimal dependencies os: ubuntu-latest - python-version: 3.7 - toxenv: py37-test-astropy30-numpy117 + python-version: '3.8' + toxenv: py38-test - name: Python 3.9 with minimal dependencies os: ubuntu-latest python-version: '3.9' toxenv: py39-test - + - name: Python 3.10 with minimal dependencies os: ubuntu-latest python-version: '3.10' @@ -44,7 +40,7 @@ jobs: - name: Code style checks os: ubuntu-latest - python-version: 3.8 + python-version: 3.9 toxenv: codestyle steps: diff --git a/CHANGES.md b/CHANGES.md index 48c7c277..93b38d72 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,9 +1,18 @@ -1.11.0 (2023-08-10) +1.13.0 (2023-10-31) +------------------- +- Refreshed build enviornment +- We now push tags to ghcr.io instead of an internal docker registry + +1.12.0 (2023-08-10) ------------------- - Added the process_by_group keyword to stages to fix a bug that wouldn't allow grouping only by instrument - Updated the logging scheme - Ack tasks late so that they may be re-queued in the event of a worker shutdown a la kubernetes +1.11.0 (2023-08-10) +------------------- +- Added optional dark temperature scaling + 1.10.1 (2023-05-31) ------------------- - We now silently fix fits errors that can be repaired automatically diff --git a/Dockerfile b/Dockerfile index 5cbf2795..947eda52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,26 +1,26 @@ -FROM continuumio/miniconda3:4.10.3 +FROM continuumio/miniconda3:23.5.2-0 -RUN conda install -y numpy pip scipy astropy pytest mock requests ipython coverage pyyaml\ - && conda install -y -c conda-forge kombu=4.4.0 elasticsearch\<6.0.0,\>=5.0.0 pytest-astropy mysql-connector-python\ - && conda clean -y --all - -RUN pip install --no-cache-dir cython logutils lcogt_logging python-dateutil sqlalchemy\>=1.3.0b1 psycopg2-binary celery[redis]==4.3.0 \ - apscheduler ocs-ingester tenacity amqp==2.6.0 cosmic-conn +# In principle I could remove the gcc to shrink the image, but pytorch is already so large it doesn't make much difference +RUN apt-get -y update && apt-get -y install gcc && \ + apt-get autoclean && \ + rm -rf /var/lib/apt/lists/* RUN mkdir /home/archive && /usr/sbin/groupadd -g 10000 "domainusers" \ && /usr/sbin/useradd -g 10000 -d /home/archive -M -N -u 10087 archive \ && chown -R archive:domainusers /home/archive -COPY --chown=10087:10000 . /lco/banzai - -RUN apt-get -y update && apt-get -y install gcc && \ - pip install --no-cache-dir git+https://github.com/cmccully/sep.git@deblending /lco/banzai/ && \ - apt-get -y remove gcc && \ - apt-get autoclean && \ - rm -rf /var/lib/apt/lists/* - USER archive ENV HOME /home/archive WORKDIR /home/archive + +COPY environment.yaml . + +RUN . /opt/conda/etc/profile.d/conda.sh && conda env create -p /home/archive/envs/banzai -f environment.yaml --solver=libmamba + +COPY --chown=10087:10000 . /lco/banzai + +ENV PATH /home/archive/envs/banzai/bin:$PATH + +RUN /home/archive/envs/banzai/bin/pip install --no-cache-dir /lco/banzai/ diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 1793f9c8..00000000 --- a/Jenkinsfile +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env groovy - -@Library('lco-shared-libs@0.0.10') _ - -pipeline { - agent any - parameters { - booleanParam( - name: 'forceEndToEnd', - defaultValue: false, - description: 'When true, forces the end-to-end tests to always run.') - } - environment { - dockerImage = null - PROJ_NAME = projName() - GIT_DESCRIPTION = gitDescribe() - DOCKER_IMG = dockerImageName("${LCO_DOCK_REG}", "${PROJ_NAME}", "${GIT_DESCRIPTION}") - } - options { - timeout(time: 8, unit: 'HOURS') - lock resource: 'BANZAILock' - } - stages { - stage('Build image') { - steps { - script { - dockerImage = docker.build("${DOCKER_IMG}", ".") - } - } - } - stage('Push image') { - steps { - script { - dockerImage.push("${GIT_DESCRIPTION}") - } - } - } - stage('Unit Tests') { - steps { - script { - sh 'docker run --rm "${DOCKER_IMG}" pytest --pyargs banzai.tests -m \'not e2e\'' - } - } - } - stage('DeployProdStack') { - agent { - label 'helm' - } - when { - buildingTag(); - } - steps { - script { - withKubeConfig([credentialsId: "prod-kube-config"]) { - sh('helm repo update && helm dependency update helm-chart/banzai/ '+ - '&& helm package helm-chart/banzai --app-version="${GIT_DESCRIPTION}" --version="${GIT_DESCRIPTION}" ' + - '&& helm upgrade --install banzai banzai-"${GIT_DESCRIPTION}".tgz --namespace=prod ' + - '--set image.tag="${GIT_DESCRIPTION}" --values=helm-chart/banzai/values-prod.yaml ' + - '--force --atomic --timeout=3600s') - } - } - } - } - stage('DeployTestStack') { - when { - anyOf { - branch 'PR-*' - expression { return params.forceEndToEnd } - } - } - steps { - script { - withKubeConfig([credentialsId: 'build-kube-config']) { - // delete previous run if the previous failed somehow - sh('kubectl -n build delete pod banzai-e2e-test || true') - // we will be testing the image that we just built - sh('sed -i -e "s^@BANZAI_IMAGE@^${DOCKER_IMG}^g" banzai/tests/e2e-k8s.yaml') - // deploy the test pod to the cluster - sh('kubectl -n build apply -f banzai/tests/e2e-k8s.yaml') - // wait for the test pod to be running - sh('kubectl -n build wait --for=condition=Ready --timeout=60m pod/banzai-e2e-test') - } - } - } - } - stage('Test-Master-Bias-Creation') { - environment { - // store stage start time in the environment so it has stage scope - START_TIME = sh(script: 'date +%s', returnStdout: true).trim() - } - when { - anyOf { - branch 'PR-*' - expression { return params.forceEndToEnd } - } - } - steps { - script { - withKubeConfig([credentialsId: 'build-kube-config']) { - sh('kubectl -n build exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias') - } - } - } - post { - always { - script { - withKubeConfig([credentialsId: 'build-kube-config']) { - env.LOGS_SINCE = sh(script: 'expr `date +%s` - ${START_TIME}', returnStdout: true).trim() - sh('kubectl -n build logs --since=${LOGS_SINCE}s --all-containers banzai-e2e-test') - sh('kubectl -n build cp -c banzai-listener banzai-e2e-test:/archive/engineering/pytest-master-bias.xml pytest-master-bias.xml') - junit 'pytest-master-bias.xml' - } - } - } - } - } - stage('Test-Master-Dark-Creation') { - environment { - // store stage start time in the environment so it has stage scope - START_TIME = sh(script: 'date +%s', returnStdout: true).trim() - } - when { - anyOf { - branch 'PR-*' - expression { return params.forceEndToEnd } - } - } - steps { - script { - withKubeConfig([credentialsId: 'build-kube-config']) { - sh('kubectl -n build exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-master-dark.xml -m master_dark') - } - } - } - post { - always { - script { - withKubeConfig([credentialsId: 'build-kube-config']) { - env.LOGS_SINCE = sh(script: 'expr `date +%s` - ${START_TIME}', returnStdout: true).trim() - sh('kubectl -n build logs --since=${LOGS_SINCE}s --all-containers banzai-e2e-test') - sh('kubectl -n build cp -c banzai-listener banzai-e2e-test:/archive/engineering/pytest-master-dark.xml pytest-master-dark.xml') - junit 'pytest-master-dark.xml' - } - } - } - } - } - stage('Test-Master-Flat-Creation') { - environment { - // store stage start time in the environment so it has stage scope - START_TIME = sh(script: 'date +%s', returnStdout: true).trim() - } - when { - anyOf { - branch 'PR-*' - expression { return params.forceEndToEnd } - } - } - steps { - script { - withKubeConfig([credentialsId: 'build-kube-config']) { - sh('kubectl -n build exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-master-flat.xml -m master_flat') - } - } - } - post { - always { - script { - withKubeConfig([credentialsId: 'build-kube-config']) { - env.LOGS_SINCE = sh(script: 'expr `date +%s` - ${START_TIME}', returnStdout: true).trim() - sh('kubectl -n build logs --since=${LOGS_SINCE}s --all-containers banzai-e2e-test') - sh('kubectl -n build cp -c banzai-listener banzai-e2e-test:/archive/engineering/pytest-master-flat.xml pytest-master-flat.xml') - junit 'pytest-master-flat.xml' - } - } - } - } - } - stage('Test-Science-File-Creation') { - environment { - // store stage start time in the environment so it has stage scope - START_TIME = sh(script: 'date +%s', returnStdout: true).trim() - } - when { - anyOf { - branch 'PR-*' - expression { return params.forceEndToEnd } - } - } - steps { - script { - withKubeConfig([credentialsId: 'build-kube-config']) { - sh('kubectl -n build exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-science-files.xml -m science_files') - } - } - } - post { - always { - script { - withKubeConfig([credentialsId: 'build-kube-config']) { - env.LOGS_SINCE = sh(script: 'expr `date +%s` - ${START_TIME}', returnStdout: true).trim() - sh('kubectl -n build logs --since=${LOGS_SINCE}s --all-containers banzai-e2e-test') - sh('kubectl -n build cp -c banzai-listener banzai-e2e-test:/archive/engineering/pytest-science-files.xml pytest-science-files.xml ') - junit 'pytest-science-files.xml' - } - } - } - success { - script { - withKubeConfig([credentialsId: 'build-kube-config']) { - sh('kubectl -n build delete pod banzai-e2e-test || true') - } - } - } - } - } - } -} - diff --git a/banzai/tests/e2e-k8s.yaml b/banzai/tests/e2e-k8s.yaml index 9fcfa266..fac1ccec 100644 --- a/banzai/tests/e2e-k8s.yaml +++ b/banzai/tests/e2e-k8s.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: Pod metadata: name: banzai-e2e-test - namespace: build labels: app.kubernetes.io/name: banzai spec: @@ -62,7 +61,7 @@ spec: initialDelaySeconds: 5 periodSeconds: 1 - name: banzai-fits-exchange - image: rabbitmq:3.7.9 + image: rabbitmq:3.12.3 imagePullPolicy: IfNotPresent resources: requests: @@ -80,7 +79,7 @@ spec: periodSeconds: 1 timeoutSeconds: 10 - name: banzai-celery-workers - image: @BANZAI_IMAGE@ + image: banzai:test-latest imagePullPolicy: IfNotPresent volumeMounts: - name: banzai-data @@ -101,7 +100,7 @@ spec: - name: OBSERVATION_PORTAL_URL value: "http://internal-observation-portal.lco.gtn/api/observations/" - name: OMP_NUM_THREADS - value: "4" + value: "2" - name: FITS_EXCHANGE value: "fits_files" - name: OPENTSDB_PYTHON_METRICS_TEST_MODE @@ -118,9 +117,9 @@ spec: - --hostname - "banzai-celery-worker" - --concurrency - - "4" + - "2" - -l - - "debug" + - "info" - "-Q" - "$(CELERY_TASK_QUEUE_NAME)" readinessProbe: @@ -144,7 +143,7 @@ spec: cpu: 8 memory: 8Gi - name: banzai-celery-beat - image: @BANZAI_IMAGE@ + image: banzai:test-latest imagePullPolicy: IfNotPresent volumeMounts: - name: banzai-data @@ -170,7 +169,7 @@ spec: cpu: 1 memory: 1Gi - name: banzai-listener - image: @BANZAI_IMAGE@ + image: banzai:test-latest imagePullPolicy: IfNotPresent volumeMounts: - name: banzai-data diff --git a/environment.yaml b/environment.yaml new file mode 100644 index 00000000..70d7b481 --- /dev/null +++ b/environment.yaml @@ -0,0 +1,316 @@ +# This environment was produced with the following: +# conda create -n banzai python=3.10 'cython<3' 'numpy<1.24' bottleneck scipy astropy pytest mock requests ipython coverage pyyaml kombu sep +'elasticsearch<6.0.0,>=5.0.0' pytest-astropy mysql-connector-python photutils psycopg2-binary tenacity 'amqp<3' 'celery[redis]>=4.3.1,<5' scikit-image emcee +python-dateutil 'sqlalchemy>=1.3.0b1' psycopg2-binary apscheduler 'pytorch>=1.6.0' --channel conda-forge --channel astropy --channel pytorch --solver=libmamba +channels: + - pytorch + - conda-forge + - astropy + - defaults +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - affine=2.4.0=pyhd8ed1ab_0 + - amqp=2.6.1=pyh9f0ad1d_0 + - anyio=4.0.0=pyhd8ed1ab_0 + - aom=3.6.1=h59595ed_0 + - apscheduler=3.10.4=pyhd8ed1ab_0 + - asdf=2.15.1=pyhd8ed1ab_0 + - asdf-astropy=0.4.0=pyhd8ed1ab_1 + - asdf-coordinates-schemas=0.2.0=pyhd8ed1ab_1 + - asdf-standard=1.0.3=pyhd8ed1ab_0 + - asdf-transform-schemas=0.3.0=pyhd8ed1ab_0 + - asdf-unit-schemas=0.1.0=pyhd8ed1ab_0 + - asdf-wcs-schemas=0.1.1=pyhd8ed1ab_0 + - astropy=5.3.4=py310h1f7b6fc_2 + - asttokens=2.4.0=pyhd8ed1ab_0 + - attrs=23.1.0=pyh71513ae_1 + - backcall=0.2.0=pyh9f0ad1d_0 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 + - backports.zoneinfo=0.2.1=py310hff52083_8 + - billiard=3.6.4.0=py310h5764c6d_3 + - blosc=1.21.5=h0f2a231_0 + - bottleneck=1.3.7=py310h1f7b6fc_1 + - brotli=1.1.0=hd590300_1 + - brotli-bin=1.1.0=hd590300_1 + - brotli-python=1.1.0=py310hc6cd4ac_1 + - brunsli=0.1=h9c3ff4c_0 + - bzip2=1.0.8=h7f98852_4 + - c-ares=1.20.1=hd590300_0 + - c-blosc2=2.10.5=hb4ffafa_0 + - ca-certificates=2023.7.22=hbcca054_0 + - cairo=1.18.0=h3faef2a_0 + - celery=4.4.0=py_0 + - certifi=2023.7.22=pyhd8ed1ab_0 + - cffi=1.16.0=py310h2fee648_0 + - cfitsio=4.3.0=hbdc6101_0 + - charls=2.4.2=h59595ed_0 + - charset-normalizer=3.3.0=pyhd8ed1ab_0 + - click=8.1.7=unix_pyh707e725_0 + - click-plugins=1.1.1=py_0 + - cligj=0.7.2=pyhd8ed1ab_1 + - colorama=0.4.6=pyhd8ed1ab_0 + - contourpy=1.1.1=py310hd41b1e2_1 + - coverage=7.3.2=py310h2372a71_0 + - cryptography=41.0.4=py310h75e40e8_0 + - cycler=0.12.1=pyhd8ed1ab_0 + - cython=0.29.36=py310hc6cd4ac_1 + - dav1d=1.2.1=hd590300_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - dnspython=2.4.2=pyhd8ed1ab_0 + - elasticsearch=5.5.3=pyh9f0ad1d_0 + - emcee=3.1.4=pyhd8ed1ab_0 + - exceptiongroup=1.1.3=pyhd8ed1ab_0 + - executing=1.2.0=pyhd8ed1ab_0 + - expat=2.5.0=hcb278e6_1 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=hab24e00_0 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.43.1=py310h2372a71_0 + - freetype=2.12.1=h267a509_2 + - freexl=2.0.0=h743c826_0 + - geos=3.12.0=h59595ed_0 + - geotiff=1.7.1=hf074850_14 + - gettext=0.21.1=h27087fc_0 + - giflib=5.2.1=h0b41bf4_3 + - greenlet=3.0.0=py310hc6cd4ac_1 + - gwcs=0.19.0=pyhd8ed1ab_0 + - h11=0.14.0=pyhd8ed1ab_0 + - h2=4.1.0=pyhd8ed1ab_0 + - hdf4=4.2.15=h2a13503_7 + - hdf5=1.14.2=nompi_h4f84152_100 + - hpack=4.0.0=pyh9f0ad1d_0 + - httpcore=1.0.0=pyhd8ed1ab_0 + - hyperframe=6.0.1=pyhd8ed1ab_0 + - hypothesis=6.87.4=pyha770c72_0 + - icu=73.2=h59595ed_0 + - idna=3.4=pyhd8ed1ab_0 + - imagecodecs=2023.9.18=py310h496a806_2 + - imageio=2.31.5=pyh8c1a49c_0 + - importlib-metadata=6.8.0=pyha770c72_0 + - importlib-resources=6.1.0=pyhd8ed1ab_0 + - importlib_resources=6.1.0=pyhd8ed1ab_0 + - iniconfig=2.0.0=pyhd8ed1ab_0 + - intel-openmp=2022.1.0=h9e868ea_3769 + - ipython=8.16.1=pyh0d859eb_0 + - jedi=0.19.1=pyhd8ed1ab_0 + - jmespath=1.0.1=pyhd8ed1ab_0 + - joblib=1.3.2=pyhd8ed1ab_0 + - json-c=0.17=h7ab15ed_0 + - jsonschema=4.19.1=pyhd8ed1ab_0 + - jsonschema-specifications=2023.7.1=pyhd8ed1ab_0 + - jxrlib=1.1=h7f98852_2 + - kealib=1.5.2=hcd42e92_1 + - keyutils=1.6.1=h166bdaf_0 + - kiwisolver=1.4.5=py310hd41b1e2_1 + - krb5=1.21.2=h659d440_0 + - lazy_loader=0.3=pyhd8ed1ab_0 + - lcms2=2.15=hb7c19ff_3 + - ld_impl_linux-64=2.40=h41732ed_0 + - lerc=4.0.0=h27087fc_0 + - libaec=1.1.2=h59595ed_1 + - libarchive=3.7.2=h039dbb9_0 + - libavif16=1.0.1=h87da1f6_2 + - libblas=3.9.0=18_linux64_openblas + - libboost-headers=1.82.0=ha770c72_6 + - libbrotlicommon=1.1.0=hd590300_1 + - libbrotlidec=1.1.0=hd590300_1 + - libbrotlienc=1.1.0=hd590300_1 + - libcblas=3.9.0=18_linux64_openblas + - libcurl=8.4.0=hca28451_0 + - libdeflate=1.19=hd590300_0 + - libedit=3.1.20191231=he28a2e2_2 + - libev=4.33=h516909a_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.2=h7f98852_5 + - libgcc-ng=13.2.0=h807b86a_2 + - libgdal=3.7.2=h6f3d308_7 + - libgfortran-ng=13.2.0=h69a702a_2 + - libgfortran5=13.2.0=ha4646dd_2 + - libglib=2.78.0=hebfc3b9_0 + - libgomp=13.2.0=h807b86a_2 + - libiconv=1.17=h166bdaf_0 + - libjpeg-turbo=3.0.0=hd590300_1 + - libkml=1.3.0=h01aab08_1018 + - liblapack=3.9.0=18_linux64_openblas + - libnetcdf=4.9.2=nompi_h80fb2b6_112 + - libnghttp2=1.52.0=h61bc06f_0 + - libnsl=2.0.0=hd590300_1 + - libopenblas=0.3.24=pthreads_h413a1c8_0 + - libpng=1.6.39=h753d276_0 + - libpq=16.0=hfc447b1_1 + - libprotobuf=3.20.3=h3eb15da_0 + - librttopo=1.1.0=hb58d41b_14 + - libspatialite=5.1.0=h090f1da_0 + - libsqlite=3.43.2=h2797004_0 + - libssh2=1.11.0=h0841786_0 + - libstdcxx-ng=13.2.0=h7e041cc_2 + - libtiff=4.6.0=ha9c0a0a_2 + - libuuid=2.38.1=h0b41bf4_0 + - libwebp-base=1.3.2=hd590300_0 + - libxcb=1.15=h0b41bf4_0 + - libxml2=2.11.5=h232c23b_1 + - libzip=1.10.1=h2629f0a_3 + - libzlib=1.2.13=hd590300_5 + - libzopfli=1.0.3=h9c3ff4c_0 + - lz4-c=1.9.4=hcb278e6_0 + - lzo=2.10=h516909a_1000 + - matplotlib-base=3.8.0=py310h62c0568_2 + - matplotlib-inline=0.1.6=pyhd8ed1ab_0 + - minizip=4.0.1=h0ab5242_5 + - mkl=2022.1.0=hc2b9512_224 + - mock=5.1.0=pyhd8ed1ab_0 + - munkres=1.1.4=pyh9f0ad1d_0 + - mysql-common=8.0.33=hf1915f5_5 + - mysql-connector-python=8.0.32=py310h6eefaca_0 + - mysql-libs=8.0.33=hca2cd23_5 + - ncurses=6.4=hcb278e6_0 + - networkx=3.1=pyhd8ed1ab_0 + - ninja=1.11.1=h924138e_0 + - nspr=4.35=h27087fc_0 + - nss=3.94=h1d7d5a4_0 + - numpy=1.23.5=py310h53a5b5f_0 + - openjpeg=2.5.0=h488ebb8_3 + - openssl=3.1.3=hd590300_0 + - packaging=23.2=pyhd8ed1ab_0 + - parso=0.8.3=pyhd8ed1ab_0 + - pcre2=10.40=hc3806b6_0 + - pexpect=4.8.0=pyh1a96a4e_2 + - photutils=1.9.0=py310h2372a71_0 + - pickleshare=0.7.5=py_1003 + - pillow=10.0.1=py310h01dd4db_2 + - pip=23.2.1=pyhd8ed1ab_0 + - pixman=0.42.2=h59595ed_0 + - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1 + - pluggy=1.3.0=pyhd8ed1ab_0 + - poppler=23.10.0=h590f24d_0 + - poppler-data=0.4.12=hd8ed1ab_0 + - postgresql=16.0=h8972f4a_1 + - proj=9.3.0=h1d62c97_1 + - prompt-toolkit=3.0.39=pyha770c72_0 + - prompt_toolkit=3.0.39=hd8ed1ab_0 + - protobuf=3.20.3=py310heca2aa9_1 + - psutil=5.9.5=py310h2372a71_1 + - psycopg2=2.9.7=py310h275853b_1 + - psycopg2-binary=2.9.7=pyhd8ed1ab_1 + - pthread-stubs=0.4=h36c2ea0_1001 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.2=pyhd8ed1ab_0 + - pycparser=2.21=pyhd8ed1ab_0 + - pyerfa=2.0.0.3=py310h1f7b6fc_1 + - pygments=2.16.1=pyhd8ed1ab_0 + - pyparsing=3.1.1=pyhd8ed1ab_0 + - pysocks=1.7.1=pyha2e5f31_6 + - pytest=7.4.2=pyhd8ed1ab_0 + - pytest-arraydiff=0.5.0=pyhd8ed1ab_0 + - pytest-astropy=0.10.0=pyhd8ed1ab_0 + - pytest-astropy-header=0.2.2=pyhd8ed1ab_0 + - pytest-cov=4.1.0=pyhd8ed1ab_0 + - pytest-doctestplus=1.0.0=pyhd8ed1ab_0 + - pytest-filter-subpackage=0.1.2=pyhd8ed1ab_0 + - pytest-mock=3.11.1=pyhd8ed1ab_0 + - pytest-openfiles=0.5.0=py_0 + - pytest-remotedata=0.4.1=pyhd8ed1ab_0 + - python=3.10.12=hd12c33a_0_cpython + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python_abi=3.10=4_cp310 + - pytorch=1.12.1=cpu_py310h75c9ab6_0 + - pytz=2023.3.post1=pyhd8ed1ab_0 + - pywavelets=1.4.1=py310h1f7b6fc_1 + - pyyaml=6.0.1=py310h2372a71_1 + - rasterio=1.3.8=py310h6a913dc_4 + - rav1e=0.6.6=he8a937b_2 + - readline=8.2=h8228510_1 + - referencing=0.30.2=pyhd8ed1ab_0 + - requests=2.31.0=pyhd8ed1ab_0 + - rpds-py=0.10.6=py310hcb5633a_0 + - scikit-image=0.22.0=py310hcc13569_2 + - scikit-learn=1.3.1=py310h1fdf081_1 + - scipy=1.11.3=py310hb13e2d6_1 + - semantic_version=2.10.0=pyhd8ed1ab_0 + - sep=1.2.1=py310h1f7b6fc_2 + - setuptools=68.2.2=pyhd8ed1ab_0 + - shapely=2.0.2=py310h7dcad9a_0 + - simplejson=3.19.2=py310h2372a71_0 + - six=1.16.0=pyh6c4a22f_0 + - sleef=3.5.1=h9b69904_2 + - snappy=1.1.10=h9fff704_0 + - sniffio=1.3.0=pyhd8ed1ab_0 + - snuggs=1.4.7=py_0 + - sortedcontainers=2.4.0=pyhd8ed1ab_0 + - sqlalchemy=2.0.22=py310h2372a71_0 + - sqlite=3.43.2=h2c6b66d_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - svt-av1=1.7.0=h59595ed_0 + - tenacity=8.2.3=pyhd8ed1ab_0 + - threadpoolctl=3.2.0=pyha21a80b_0 + - tifffile=2023.9.26=pyhd8ed1ab_0 + - tiledb=2.16.3=hf0b6e87_3 + - tk=8.6.13=h2797004_0 + - toml=0.10.2=pyhd8ed1ab_0 + - tomli=2.0.1=pyhd8ed1ab_0 + - tqdm=4.66.1=pyhd8ed1ab_0 + - traitlets=5.11.2=pyhd8ed1ab_0 + - typing-extensions=4.8.0=hd8ed1ab_0 + - typing_extensions=4.8.0=pyha770c72_0 + - tzcode=2023c=h0b41bf4_0 + - tzdata=2023c=h71feb2d_0 + - tzlocal=5.1=py310hff52083_0 + - unicodedata2=15.1.0=py310h2372a71_0 + - uriparser=0.9.7=hcb278e6_1 + - wcwidth=0.2.8=pyhd8ed1ab_0 + - wheel=0.41.2=pyhd8ed1ab_0 + - xerces-c=3.2.4=hac6953d_3 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.7=h8ee46fc_0 + - xorg-libxau=1.0.11=hd590300_0 + - xorg-libxdmcp=1.1.3=h7f98852_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xz=5.2.6=h166bdaf_0 + - yaml=0.2.5=h7f98852_2 + - zfp=1.0.0=h59595ed_4 + - zipp=3.17.0=pyhd8ed1ab_0 + - zlib=1.2.13=hd590300_5 + - zlib-ng=2.0.7=h0b41bf4_0 + - zstd=1.5.5=hfc55251_0 + - pip: + - asciitree==0.3.3 + - astropy-healpix==1.0.0 + - async-timeout==4.0.3 + - boto3==1.28.63 + - botocore==1.31.63 + - cloudpickle==2.2.1 + - cosmic-conn==0.4.1 + - dask==2023.9.3 + - fasteners==0.19 + - fsspec==2023.9.2 + - kombu==4.6.11 + - lcogt-logging==0.3.2 + - locket==1.0.0 + - logutils==0.3.5 + - numcodecs==0.12.0 + - ocs-archive==0.2.10 + - ocs-ingester==3.0.5 + - opensearch-py==1.1.0 + - opentsdb-http-client==0.2.0 + - opentsdb-python-metrics==0.2.0 + - partd==1.4.1 + - pretty-errors==1.2.25 + - redis==5.0.1 + - reproject==0.12.0 + - s3transfer==0.7.0 + - toolz==0.12.0 + - urllib3==1.26.17 + - vine==1.3.0 + - zarr==2.16.1 diff --git a/setup.cfg b/setup.cfg index 3a8e574f..1fe5c641 100755 --- a/setup.cfg +++ b/setup.cfg @@ -108,20 +108,21 @@ install_requires = numpy<1.24 cython mysql-connector-python - lcogt_logging==0.3.2 - sep - kombu==4.4.0 - amqp==2.6.0 + lcogt_logging + photutils + bottleneck + kombu + amqp requests - opensearch-py==1.0.0 + opensearch-py>=1,<2 pytest>=4.0 pyyaml psycopg2-binary - celery[redis]==4.3.1 + celery[redis]>=4.3.1,<5 apscheduler python-dateutil ocs_ingester>=3.0.4,<4.0.0 - tenacity==6.0.0 + tenacity>=8,<=9 python-dateutil emcee scikit-image