From 1db7fa7cdf582870a2ce025921a5dd605022db59 Mon Sep 17 00:00:00 2001 From: Juan Escudero Pedrosa Date: Wed, 10 Apr 2024 16:36:42 +0200 Subject: [PATCH 01/14] trigger only on tagged releases --- .github/workflows/docs.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 33f0a71d..25b69941 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -2,8 +2,6 @@ name: docs on: push: - branches: - - main tags: - 'v*' From 5503caf760d4809876e9436e9ffe6b54bda7950a Mon Sep 17 00:00:00 2001 From: Juan Escudero Pedrosa Date: Wed, 10 Apr 2024 17:01:06 +0200 Subject: [PATCH 02/14] fix missing data in docs notebooks --- .github/workflows/ci.yml | 3 +++ .github/workflows/docs.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16ca3c28..98bc3cc1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -130,5 +130,8 @@ jobs: python manage.py migrate --no-input ls -lh $HOME/.iop4data/ + - name: Run iop4 on the test data + run: iop4 --list-local + - name: Build docs run: make docs-sphinx \ No newline at end of file diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 25b69941..61c66a88 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -71,6 +71,9 @@ jobs: python manage.py migrate --no-input ls -lh $HOME/.iop4data/ + - name: Run iop4 on the test data + run: iop4 --list-local + - name: Build docs run: make docs-sphinx From 1aa6b1cb04f5c324e76e25a84a1dbbd6530e5b6a Mon Sep 17 00:00:00 2001 From: Juan Escudero Pedrosa Date: Wed, 10 Apr 2024 17:08:48 +0200 Subject: [PATCH 03/14] omit unnecessary run --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98bc3cc1..e0b4d45c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,6 +86,8 @@ jobs: docs: runs-on: [self-hosted, x64, Linux, ubuntu, generic] + # Don't run if this is a tag push, already done during docs deployment (docs.yml) + if: ${{ !startsWith(github.ref, 'refs/tags/v') }} steps: - uses: actions/checkout@v4 From 26caf81cfa7784598090a0b799907b49411308e3 Mon Sep 17 00:00:00 2001 From: Juan Escudero Pedrosa Date: Wed, 10 Apr 2024 17:36:33 +0200 Subject: [PATCH 04/14] create logs subdir if it does not exist --- iop4lib/config.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/iop4lib/config.py b/iop4lib/config.py index 66542127..c376221c 100644 --- a/iop4lib/config.py +++ b/iop4lib/config.py @@ -139,6 +139,11 @@ def configure(self, config_path=None, config_db=False, config_logging=True, gono if not os.path.exists(self.datadir): os.makedirs(self.datadir) + # Check if the logs subdirectory exists, if not, create it + + if not os.path.exists(os.path.join(self.datadir, "logs")): + os.makedirs(os.path.join(self.datadir, "logs")) + # Load OSN names from external file if indicated, load them into patterns like name*.fit, name*.fits, or name*.fts. if self.osn_source_list_path is not None and os.path.exists(self.osn_source_list_path): From e7dbd691327de41c062a9dc60263b7087d6bd507 Mon Sep 17 00:00:00 2001 From: Juan Escudero Pedrosa Date: Wed, 10 Apr 2024 22:15:11 +0200 Subject: [PATCH 05/14] show debug messages --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0b4d45c..1467db3c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -133,7 +133,7 @@ jobs: ls -lh $HOME/.iop4data/ - name: Run iop4 on the test data - run: iop4 --list-local + run: iop4 --list-local -o log_level=10 - name: Build docs run: make docs-sphinx \ No newline at end of file From bbbe8f705224e02151441b24c0318428c0641230 Mon Sep 17 00:00:00 2001 From: Juan Escudero Pedrosa Date: Wed, 10 Apr 2024 23:08:45 +0200 Subject: [PATCH 06/14] fix: log_level did not admit an int --- iop4lib/iop4.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iop4lib/iop4.py b/iop4lib/iop4.py index 190bf94c..6a7ebfc1 100644 --- a/iop4lib/iop4.py +++ b/iop4lib/iop4.py @@ -232,7 +232,7 @@ def parse_config_overrides(overrides): # correct the type of the config options for k, v in config.items(): # interpret the log_level option - if k == "log_level": + if k == "log_level" and not v.isnumeric(): v = config[k] = getattr(logging, v.upper()) # check the type of the config option, if it is an int, convert it if k in iop4conf and isinstance(iop4conf[k], int): From c0db145927aad77d9ef63c069b088c3448ce81a9 Mon Sep 17 00:00:00 2001 From: Juan Escudero Pedrosa Date: Wed, 10 Apr 2024 23:25:56 +0200 Subject: [PATCH 07/14] how mem usage --- .github/workflows/ci.yml | 2 ++ config/config.example.yaml | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1467db3c..f4ef84ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,8 @@ jobs: tests: runs-on: [self-hosted, x64, Linux, ubuntu, generic] + # for testing docs: + if: false strategy: max-parallel: 1 matrix: diff --git a/config/config.example.yaml b/config/config.example.yaml index 4bbdea89..bb0eeff9 100644 --- a/config/config.example.yaml +++ b/config/config.example.yaml @@ -27,7 +27,8 @@ mplt_default_dpi: 100 # dpi for matplotlib (e.g. 100) log_file: ~/.iop4data/logs/iop4.log # Path to log file. log_date_format: '%Y-%m-%d %H:%M:%S' # -log_format: '%(asctime)s - %(name)s [%(filename)s:%(lineno)d] - %(levelname)s - %(message)s' # You can also add '%(process)d (%(proc_memory)s)' to the format to show pid and memory usage per process. +#log_format: '%(asctime)s - %(name)s [%(filename)s:%(lineno)d] - %(levelname)s - %(message)s' # You can also add '%(process)d (%(proc_memory)s)' to the format to show pid and memory usage per process. +log_format: '%(asctime)s - %(process)d (%(proc_memory)s) [%(filename)s:%(lineno)d] - %(levelname)s - %(message)s' # You can also add '%(process)d (%(proc_memory)s)' to the format to show pid and memory usage per process. log_level: 20 # . Possible values are: 10 (DEBUG), 20 (INFO), 30 (WARNING), 40 (ERROR), 50 (CRITICAL). ############ From 748d9f84d25039540a394920da92e3277465d716 Mon Sep 17 00:00:00 2001 From: Juan Escudero Pedrosa Date: Wed, 10 Apr 2024 23:58:53 +0200 Subject: [PATCH 08/14] try again --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4ef84ac..5360424b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -124,6 +124,7 @@ jobs: - name: Extract test data in the default data folder run: | tar -xzf $HOME/iop4testdata.tar.gz -C $HOME + rm $HOME/iop4testdata.tar.gz mv $HOME/iop4testdata $HOME/.iop4data - name: Create the DB From a501c3b6314d196f9384a91ac2368b44bcd1736e Mon Sep 17 00:00:00 2001 From: Juan Escudero Pedrosa Date: Thu, 11 Apr 2024 00:54:05 +0200 Subject: [PATCH 09/14] missing mount of astrometry index files --- .github/workflows/ci.yml | 8 +++++++- .github/workflows/docs.yml | 7 +++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5360424b..63f12627 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,6 +114,13 @@ jobs: pip install -e .[doc] python -c 'import iop4lib; print(iop4lib.__version__)' + - name: Use mounted astrometry index files + run: | + ln -s /mnt/astrometry_cache $HOME/.astrometry_cache + + - name: Check astrometry index files were correctly mounted + run: ls $HOME/.astrometry_cache/5200/index-5200-00.fits + - name: Download test data env: TEST_DATA_PASSWORD: ${{ secrets.test_data_password }} @@ -124,7 +131,6 @@ jobs: - name: Extract test data in the default data folder run: | tar -xzf $HOME/iop4testdata.tar.gz -C $HOME - rm $HOME/iop4testdata.tar.gz mv $HOME/iop4testdata $HOME/.iop4data - name: Create the DB diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 61c66a88..3360563c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -51,6 +51,13 @@ jobs: pip install -e .[doc] python -c 'import iop4lib; print(iop4lib.__version__)' + - name: Use mounted astrometry index files + run: | + ln -s /mnt/astrometry_cache $HOME/.astrometry_cache + + - name: Check astrometry index files were correctly mounted + run: ls $HOME/.astrometry_cache/5200/index-5200-00.fits + - name: Download test data env: TEST_DATA_PASSWORD: ${{ secrets.test_data_password }} From 6185461b07d0d960c7c33ab1410034f1a6efa837 Mon Sep 17 00:00:00 2001 From: Juan Escudero Pedrosa Date: Thu, 11 Apr 2024 02:21:30 +0200 Subject: [PATCH 10/14] all ci --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63f12627..014561d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,8 +37,6 @@ jobs: tests: runs-on: [self-hosted, x64, Linux, ubuntu, generic] - # for testing docs: - if: false strategy: max-parallel: 1 matrix: From 41b67fe204b458a150071c48305eea00fc074a65 Mon Sep 17 00:00:00 2001 From: Juan Escudero Pedrosa Date: Thu, 11 Apr 2024 02:47:16 +0200 Subject: [PATCH 11/14] revert log config --- config/config.example.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/config.example.yaml b/config/config.example.yaml index bb0eeff9..4bbdea89 100644 --- a/config/config.example.yaml +++ b/config/config.example.yaml @@ -27,8 +27,7 @@ mplt_default_dpi: 100 # dpi for matplotlib (e.g. 100) log_file: ~/.iop4data/logs/iop4.log # Path to log file. log_date_format: '%Y-%m-%d %H:%M:%S' # -#log_format: '%(asctime)s - %(name)s [%(filename)s:%(lineno)d] - %(levelname)s - %(message)s' # You can also add '%(process)d (%(proc_memory)s)' to the format to show pid and memory usage per process. -log_format: '%(asctime)s - %(process)d (%(proc_memory)s) [%(filename)s:%(lineno)d] - %(levelname)s - %(message)s' # You can also add '%(process)d (%(proc_memory)s)' to the format to show pid and memory usage per process. +log_format: '%(asctime)s - %(name)s [%(filename)s:%(lineno)d] - %(levelname)s - %(message)s' # You can also add '%(process)d (%(proc_memory)s)' to the format to show pid and memory usage per process. log_level: 20 # . Possible values are: 10 (DEBUG), 20 (INFO), 30 (WARNING), 40 (ERROR), 50 (CRITICAL). ############ From df3070f02169d0d8701f1747e2dd97e1a10afa2e Mon Sep 17 00:00:00 2001 From: Juan Escudero Pedrosa Date: Thu, 11 Apr 2024 03:01:00 +0200 Subject: [PATCH 12/14] remove debug logging --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 014561d6..047b23ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,7 +140,7 @@ jobs: ls -lh $HOME/.iop4data/ - name: Run iop4 on the test data - run: iop4 --list-local -o log_level=10 + run: iop4 --list-local - name: Build docs run: make docs-sphinx \ No newline at end of file From 41de24be4e4cc8a7994f2d4d92cf1483061f609c Mon Sep 17 00:00:00 2001 From: Juan Escudero Pedrosa Date: Thu, 11 Apr 2024 12:54:46 +0200 Subject: [PATCH 13/14] check if overwrite_input improves memory usage --- .github/workflows/ci.yml | 2 +- iop4lib/db/masterdark.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 047b23ec..19c61c57 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,7 +140,7 @@ jobs: ls -lh $HOME/.iop4data/ - name: Run iop4 on the test data - run: iop4 --list-local + run: iop4 --list-local -o log_level=debug - name: Build docs run: make docs-sphinx \ No newline at end of file diff --git a/iop4lib/db/masterdark.py b/iop4lib/db/masterdark.py index fce1d387..7317edb1 100644 --- a/iop4lib/db/masterdark.py +++ b/iop4lib/db/masterdark.py @@ -198,7 +198,7 @@ def build_file(self): data = (hdul[0].data - self.masterbias.data) / self.exptime data_L.append(data) - data = np.nanmedian(data_L, axis=0) + data = np.nanmedian(data_L, axis=0, overwrite_input=True) logger.debug(f"Building header") From b9b0d034fd8fc239bb20ead3c3e8558363273050 Mon Sep 17 00:00:00 2001 From: Juan Escudero Pedrosa Date: Thu, 11 Apr 2024 13:26:09 +0200 Subject: [PATCH 14/14] Revert "check if overwrite_input improves memory usage" This reverts commit 41de24be4e4cc8a7994f2d4d92cf1483061f609c. --- .github/workflows/ci.yml | 2 +- iop4lib/db/masterdark.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19c61c57..047b23ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,7 +140,7 @@ jobs: ls -lh $HOME/.iop4data/ - name: Run iop4 on the test data - run: iop4 --list-local -o log_level=debug + run: iop4 --list-local - name: Build docs run: make docs-sphinx \ No newline at end of file diff --git a/iop4lib/db/masterdark.py b/iop4lib/db/masterdark.py index 7317edb1..fce1d387 100644 --- a/iop4lib/db/masterdark.py +++ b/iop4lib/db/masterdark.py @@ -198,7 +198,7 @@ def build_file(self): data = (hdul[0].data - self.masterbias.data) / self.exptime data_L.append(data) - data = np.nanmedian(data_L, axis=0, overwrite_input=True) + data = np.nanmedian(data_L, axis=0) logger.debug(f"Building header")