From 24526b03596065543077c711c334a797e6715600 Mon Sep 17 00:00:00 2001 From: Seongsu Jeong Date: Thu, 28 Mar 2024 15:07:08 -0700 Subject: [PATCH 1/4] Separate IONEX fetch routine for less boilerplate; code change in filename extension handling; re-try IONEX retrieval with the initial IONEX file name format does not work --- src/compass/utils/iono.py | 75 +++++++++++++++++++++++++++++++++------ 1 file changed, 64 insertions(+), 11 deletions(-) diff --git a/src/compass/utils/iono.py b/src/compass/utils/iono.py index fd3d148d..e69cdf30 100644 --- a/src/compass/utils/iono.py +++ b/src/compass/utils/iono.py @@ -17,6 +17,9 @@ import numpy as np from scipy import interpolate +# Approximatedate from which the new IONEX file name format needs to be used +# https://cddis.nasa.gov/Data_and_Derived_Products/GNSS/atmospheric_products.html +NEW_IONEX_FILENAME_FORMAT_FROM = dt.datetime.fromisoformat('2023-10-18') def read_ionex(tec_file): ''' @@ -231,13 +234,37 @@ def download_ionex(date_str, tec_dir, sol_code='jpl', date_fmt='%Y%m%d'): fname_dst_uncomp: str Path to local uncompressed IONEX file ''' - # get the source (remote) and destination (local) file path/url - kwargs = dict(sol_code=sol_code, date_fmt=date_fmt) - fname_src = get_ionex_filename(date_str, tec_dir=None, **kwargs) - fname_dst = get_ionex_filename(date_str, tec_dir=tec_dir, **kwargs) + '.Z' - fname_dst_uncomp = fname_dst[:-2] - # download - compose cmd + # get the source (remote) and destination (local) file path/url + date_tec = dt.datetime.strptime(date_str, date_fmt) + + # determine the initial format to try + use_new_ionex_filename_format = date_tec >= NEW_IONEX_FILENAME_FORMAT_FROM + kwargs = dict(sol_code=sol_code, + date_fmt=date_fmt, + new_filename_format=use_new_ionex_filename_format) + try: + # Initial try with the automatically-determined IONEX file name format + fname_src = get_ionex_filename(date_str, tec_dir=None, **kwargs) + fname_dst_uncomp = get_ionex_filename(date_str, tec_dir=tec_dir, **kwargs) + ionex_zip_extension = fname_src[fname_src.rfind('.'):] + fname_dst = fname_dst_uncomp + ionex_zip_extension + + fetch_ionex_from_remote(fname_src, fname_dst) + + except RuntimeError: + logging.info('Initial download attempt was not successful. ' + 'Trying another IONEX file name format.') + kwargs['new_filename_format'] = not use_new_ionex_filename_format + + fname_src = get_ionex_filename(date_str, tec_dir=None, **kwargs) + fname_dst_uncomp = get_ionex_filename(date_str, tec_dir=tec_dir, **kwargs) + ionex_zip_extension = fname_src[fname_src.rfind('.'):] + fname_dst = fname_dst_uncomp + ionex_zip_extension + + fetch_ionex_from_remote(fname_src, fname_dst) + + """# download - compose cmd cmd = f'wget --continue --auth-no-challenge "{fname_src}"' if os.path.isfile(fname_dst) and os.path.getsize(fname_dst) > 1000: cmd += ' --timestamping' @@ -249,7 +276,7 @@ def download_ionex(date_str, tec_dir, sol_code='jpl', date_fmt='%Y%m%d'): pwd = os.getcwd() os.chdir(tec_dir) os.system(cmd) - os.chdir(pwd) + os.chdir(pwd)""" # uncompress # if output file 1) does not exist or 2) smaller than 400k in size or 3) older @@ -264,8 +291,29 @@ def download_ionex(date_str, tec_dir, sol_code='jpl', date_fmt='%Y%m%d'): return fname_dst_uncomp +def fetch_ionex_from_remote(ionex_url, ionex_local_path): + # download - compose cmd + cmd = f'wget --continue --auth-no-challenge "{ionex_url}"' + if os.path.isfile(ionex_local_path) and os.path.getsize(ionex_local_path) > 1000: + cmd += ' --timestamping' + + # Record executed command line in logging file + logging.info(f'Execute command: {cmd}') + + # download - run cmd in output dir + tec_dir = os.path.dirname(ionex_local_path) + pwd = os.getcwd() + os.chdir(tec_dir) + exit_status = os.system(cmd) + os.chdir(pwd) + + if exit_status != 0: + raise RuntimeError('wget execution was not successful') + + def get_ionex_filename(date_str, tec_dir=None, sol_code='jpl', - date_fmt='%Y%m%d'): + date_fmt='%Y%m%d', + new_filename_format=None): ''' Get the file name of the IONEX file @@ -289,15 +337,20 @@ def get_ionex_filename(date_str, tec_dir=None, sol_code='jpl', dd = dt.datetime.strptime(date_str, date_fmt) doy = f'{dd.timetuple().tm_yday:03d}' - yy = str(dd.year)[2:4] + yy_full = str(dd.year) + yy = yy_full[2:4] # file name base - fname = f"{sol_code.lower()}g{doy}0.{yy}i.Z" + + if new_filename_format: + fname = f'{sol_code.upper()}0OPSFIN_{yy_full}{doy}0000_01D_02H_GIM.INX.gz' + else: + fname = f"{sol_code.lower()}g{doy}0.{yy}i.Z" # full path if tec_dir: # local uncompressed file path - tec_file = os.path.join(tec_dir, fname[:-2]) + tec_file = os.path.join(tec_dir, fname[:fname.rfind('.')]) else: # remote compressed file path url_dir = "https://cddis.nasa.gov/archive/gnss/products/ionex" From 94f91e73bfa57bd0c9c11587436a4bc613ea2ae5 Mon Sep 17 00:00:00 2001 From: Seongsu Jeong Date: Thu, 28 Mar 2024 16:17:58 -0700 Subject: [PATCH 2/4] remove commented code and trailing whitespace --- src/compass/utils/iono.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/src/compass/utils/iono.py b/src/compass/utils/iono.py index e69cdf30..694a1405 100644 --- a/src/compass/utils/iono.py +++ b/src/compass/utils/iono.py @@ -264,20 +264,6 @@ def download_ionex(date_str, tec_dir, sol_code='jpl', date_fmt='%Y%m%d'): fetch_ionex_from_remote(fname_src, fname_dst) - """# download - compose cmd - cmd = f'wget --continue --auth-no-challenge "{fname_src}"' - if os.path.isfile(fname_dst) and os.path.getsize(fname_dst) > 1000: - cmd += ' --timestamping' - - # Record executed command line in logging file - logging.info(f'Execute command: {cmd}') - - # download - run cmd in output dir - pwd = os.getcwd() - os.chdir(tec_dir) - os.system(cmd) - os.chdir(pwd)""" - # uncompress # if output file 1) does not exist or 2) smaller than 400k in size or 3) older if (not os.path.isfile(fname_dst_uncomp) @@ -299,7 +285,7 @@ def fetch_ionex_from_remote(ionex_url, ionex_local_path): # Record executed command line in logging file logging.info(f'Execute command: {cmd}') - + # download - run cmd in output dir tec_dir = os.path.dirname(ionex_local_path) pwd = os.getcwd() From 8a6c924796e347f7688cf4498fea95c813314ca6 Mon Sep 17 00:00:00 2001 From: Seongsu Jeong Date: Tue, 2 Apr 2024 18:29:36 -0700 Subject: [PATCH 3/4] Add a routine to check if the resource exists with the URL --- src/compass/utils/helpers.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/compass/utils/helpers.py b/src/compass/utils/helpers.py index f7e49f8a..6fe2e281 100644 --- a/src/compass/utils/helpers.py +++ b/src/compass/utils/helpers.py @@ -3,6 +3,7 @@ from datetime import timedelta import itertools import os +import requests import sqlite3 import time @@ -459,3 +460,33 @@ def get_time_delta_str(t_prev: time) -> str: ''' return str(timedelta(seconds=time.perf_counter() - t_prev)).split(".", maxsplit=1)[0] + + +def check_url(url): + ''' + Check if a resource exists at the given URL. + + Parameters + ---------- + url: str + URL to the object + + Returns + ------- + _: Bool + `True` if the resource exists in the URL provide; False otherwise + ''' + + try: + response = requests.head(url, allow_redirects=True) + # A 200 OK or 30x redirect status code means the resource exists. + print('status code:',response.status_code) + if response.status_code in range(200, 400): + print(f"Resource exists at: {url}") + return True + else: + print(f"Resource does not exist at: {url}") + return False + except requests.exceptions.RequestException as e: + print(f"An error occurred: {e}") + return False From 166ca5feaaf92395ac8efc38ee86228225bd09a0 Mon Sep 17 00:00:00 2001 From: Seongsu Jeong Date: Tue, 2 Apr 2024 18:30:38 -0700 Subject: [PATCH 4/4] remove `--keep` argument in gzip command; check if the TEC file exists --- src/compass/utils/iono.py | 51 +++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/src/compass/utils/iono.py b/src/compass/utils/iono.py index 694a1405..bea80b24 100644 --- a/src/compass/utils/iono.py +++ b/src/compass/utils/iono.py @@ -16,6 +16,7 @@ import isce3 import numpy as np from scipy import interpolate +from compass.utils.helpers import check_url # Approximatedate from which the new IONEX file name format needs to be used # https://cddis.nasa.gov/Data_and_Derived_Products/GNSS/atmospheric_products.html @@ -240,9 +241,11 @@ def download_ionex(date_str, tec_dir, sol_code='jpl', date_fmt='%Y%m%d'): # determine the initial format to try use_new_ionex_filename_format = date_tec >= NEW_IONEX_FILENAME_FORMAT_FROM + kwargs = dict(sol_code=sol_code, date_fmt=date_fmt, - new_filename_format=use_new_ionex_filename_format) + new_filename_format=use_new_ionex_filename_format, + check_if_exists=False) try: # Initial try with the automatically-determined IONEX file name format fname_src = get_ionex_filename(date_str, tec_dir=None, **kwargs) @@ -270,7 +273,7 @@ def download_ionex(date_str, tec_dir, sol_code='jpl', date_fmt='%Y%m%d'): or os.path.getsize(fname_dst_uncomp) < 400e3 or os.path.getmtime(fname_dst_uncomp) < os.path.getmtime( fname_dst)): - cmd = f"gzip --force --keep --decompress {fname_dst}" + cmd = f"gzip --force --decompress {fname_dst}" logging.info(f'Execute command: {cmd}') os.system(cmd) @@ -299,7 +302,8 @@ def fetch_ionex_from_remote(ionex_url, ionex_local_path): def get_ionex_filename(date_str, tec_dir=None, sol_code='jpl', date_fmt='%Y%m%d', - new_filename_format=None): + new_filename_format=None, + check_if_exists=False): ''' Get the file name of the IONEX file @@ -314,6 +318,10 @@ def get_ionex_filename(date_str, tec_dir=None, sol_code='jpl', (values: cod, esa, igs, jpl, upc, uqr) date_fmt: str Date format string + new_file_format: bool + Flag whether not not to use the new IONEX TEC file format + check_if_exists: bool + Flag whether the IONEX file exists in the local directory or in the URL Returns ------- @@ -327,20 +335,33 @@ def get_ionex_filename(date_str, tec_dir=None, sol_code='jpl', yy = yy_full[2:4] # file name base + # Keep both the old- and new- formats of the file names + fname_list = [f"{sol_code.lower()}g{doy}0.{yy}i.Z", + f'{sol_code.upper()}0OPSFIN_{yy_full}{doy}0000_01D_02H_GIM.INX.gz'] + # Decide which file name format to try first if new_filename_format: - fname = f'{sol_code.upper()}0OPSFIN_{yy_full}{doy}0000_01D_02H_GIM.INX.gz' - else: - fname = f"{sol_code.lower()}g{doy}0.{yy}i.Z" - - # full path - if tec_dir: - # local uncompressed file path - tec_file = os.path.join(tec_dir, fname[:fname.rfind('.')]) - else: - # remote compressed file path - url_dir = "https://cddis.nasa.gov/archive/gnss/products/ionex" - tec_file = os.path.join(url_dir, str(dd.year), doy, fname) + fname_list.reverse() + + for fname in fname_list: + # This initial value in the flag will make the for loop to choose the + # first format in the list when `check_if_exists` is turned off + flag_ionex_exists = True + # full path + if tec_dir: + # local uncompressed file path + tec_file = os.path.join(tec_dir, fname[:fname.rfind('.')]) + if check_if_exists: + flag_ionex_exists = os.path.exists(tec_file) + else: + # remote compressed file path + url_dir = "https://cddis.nasa.gov/archive/gnss/products/ionex" + tec_file = os.path.join(url_dir, str(dd.year), doy, fname) + if check_if_exists: + flag_ionex_exists = check_url(tec_file) + + if flag_ionex_exists: + break return tec_file