From 25a1d871bcd39e9875c7d32f9dc82c617e1ed953 Mon Sep 17 00:00:00 2001 From: Hannah Mark Date: Fri, 1 Nov 2024 12:25:29 -0400 Subject: [PATCH 1/6] progress bar flags so they can be turned off for tests --- shipgrav/io.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/shipgrav/io.py b/shipgrav/io.py index d5e522b..d1d7779 100644 --- a/shipgrav/io.py +++ b/shipgrav/io.py @@ -17,7 +17,7 @@ ######################################################################## -def read_nav(ship, pathlist, sampling=1, talker=None, ship_function=None): +def read_nav(ship, pathlist, sampling=1, talker=None, ship_function=None, progressbar=True): """ Read navigation strings from .GPS (or similar) files. Ships have different formats and use different talkers for preferred @@ -38,6 +38,8 @@ def read_nav(ship, pathlist, sampling=1, talker=None, ship_function=None): This function should return arrays of lon, lat, and timestamps. Look at _navcoords() and navdate_Atlantis() (and similar functions) for examples. :type ship_function: function, optional + :param progressbar: display progress bar while list of files is read + :type progressbar: bool :returns: (*pd.DataFrame*) time series of geographic coordinates and timestamps """ @@ -71,7 +73,7 @@ def read_nav(ship, pathlist, sampling=1, talker=None, ship_function=None): lonlon = np.array([]) latlat = np.array([]) - for fpath in tqdm(pathlist,desc='reading nav'): # loop nav files (may be a lot of them) + for fpath in tqdm(pathlist,desc='reading nav',disable=~progressbar): # loop nav files (may be a lot of them) with open(fpath, 'r') as f: allnav = np.array(f.readlines()) # read the entire file @@ -360,7 +362,7 @@ def _clean_180cross(gps_nav): ######################################################################## -def read_bgm_rgs(fp, ship): +def read_bgm_rgs(fp, ship, progressbar=True): """Read BGM gravity from RGS files. RGS is supposedly a standard format; is consistent between Atlantis @@ -370,6 +372,8 @@ def read_bgm_rgs(fp, ship): :type fp: string, or list of strings :param ship: ship name :type ship: string + :param progressbar: display progress bar while list of files is read + :type progressbar: bool :returns: (*pd.DataFrame*) timestamps, raw gravity, and geographic coordinates """ @@ -382,7 +386,7 @@ def read_bgm_rgs(fp, ship): fp = [fp,] # make a list if only one path is given dats = [] - for path in fp: + for path in tqdm(fp, desc='reading RGS files', disable=~progressbar): dat = pd.read_csv(path, delimiter=' ', names=['date', 'time', 'grav', 'lat', 'lon'], usecols=(1, 2, 3, 11, 12)) dat['date_time'] = pd.to_datetime(dat.pop('date')+' '+dat.pop('time'),utc=True) @@ -391,7 +395,7 @@ def read_bgm_rgs(fp, ship): return pd.concat(dats, ignore_index=True) -def read_bgm_raw(fp, ship, scale=None, ship_function=None): +def read_bgm_raw(fp, ship, scale=None, ship_function=None, progressbar=True): """Read BGM gravity from raw (serial) files (not RGS). This function uses scale factors determined for specific BGM meters @@ -408,6 +412,8 @@ def read_bgm_raw(fp, ship, scale=None, ship_function=None): The function should return a pandas.DataFrame with timestamps and counts. Look at _bgmserial_Atlantis() and similar functions for examples. :type ship_function: function, optional + :param progressbar: display progress bar while list of files is read + :type progressbar: bool :return: (*pd.DataFrame*) timestamps and calibrated raw gravity values """ @@ -430,7 +436,7 @@ def read_bgm_raw(fp, ship, scale=None, ship_function=None): if type(fp) is str: fp = [fp,] # make a list if only one path is given dats = [] - for path in fp: + for path in tqdm(fp,desc='reading BGM files',disable=~progressbar): if ship_function != None: dat = ship_function(path) else: @@ -520,7 +526,7 @@ def _despike_bgm_serial(dat, thresh=8000): ######################################################################## -def read_dgs_laptop(fp, ship, ship_function=None): +def read_dgs_laptop(fp, ship, ship_function=None, progressbar=True): """Read DGS 'laptop' file(s), usually written as .dat files. :param fp: filepath(s) @@ -531,6 +537,8 @@ def read_dgs_laptop(fp, ship, ship_function=None): The function should return a pandas.DataFrame. See _dgs_laptop_general() for an example. :type ship_function: function, optional + :param progressbar: display progress bar while list of files is read + :type progressbar: bool :return: *(pd.DataFrame)* DGS output time series """ @@ -538,7 +546,7 @@ def read_dgs_laptop(fp, ship, ship_function=None): fp = [fp,] # listify dats = [] - for path in tqdm(fp,desc='reading DGS files'): + for path in tqdm(fp,desc='reading DGS files',disable=~progressbar): if ship_function is not None: dat = ship_function(path) else: @@ -577,7 +585,7 @@ def _dgs_laptop_Thompson(path): return dat -def read_dgs_raw(fp, ship, scale_ccp=True): +def read_dgs_raw(fp, ship, scale_ccp=True, progressbar=True): """Read raw (serial) output files from DGS AT1M. These will be in AD units mostly. @@ -589,6 +597,8 @@ def read_dgs_raw(fp, ship, scale_ccp=True): :type fp: string or list of strings :param ship: ship name :type ship: string + :param progressbar: display progress bar while list of files is read + :type progressbar: bool :return: (*pd.DataFrame*) DGS output time series """ @@ -596,7 +606,7 @@ def read_dgs_raw(fp, ship, scale_ccp=True): if type(fp) is str: fp = [fp,] # listify dats = [] - for path in tqdm(fp,desc='reading DGS files'): + for path in tqdm(fp,desc='reading DGS files',disable=~progressbar): if ship == 'Thompson': # always with the special file formats dat = _dgs_raw_Thompson(path) else: # there might be exceptions besides Thompson but I don't know about them yet From fdaa51524b575e9b1731233059cd8684cf567862 Mon Sep 17 00:00:00 2001 From: Hannah Mark Date: Fri, 1 Nov 2024 12:32:48 -0400 Subject: [PATCH 2/6] Revelle BGM timestamps have UTC info already, oops --- shipgrav/io.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/shipgrav/io.py b/shipgrav/io.py index d1d7779..a5f76e5 100644 --- a/shipgrav/io.py +++ b/shipgrav/io.py @@ -483,8 +483,6 @@ def _bgmserial_Revelle(path): def count(x): return (int(x.split(':')[-1])) dat = pd.read_csv(path, delimiter=' ', names=['date_time', 'counts'], usecols=(0, 1), parse_dates=[0], converters={'counts': count}) - ndt = [e.tz_localize(timezone.utc) for e in dat['date_time']] - dat['date_time'] = ndt return dat def _bgmserial_Langseth(path): From c49718632015d39013f84840fba6749833377322 Mon Sep 17 00:00:00 2001 From: Hannah Mark Date: Fri, 1 Nov 2024 14:02:21 -0400 Subject: [PATCH 3/6] fix timezone bug for Ride INGGA --- shipgrav/io.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/shipgrav/io.py b/shipgrav/io.py index a5f76e5..428ba30 100644 --- a/shipgrav/io.py +++ b/shipgrav/io.py @@ -1,7 +1,7 @@ import mmap import os import re -from datetime import datetime, timezone +from datetime import datetime, timezone, UTC import numpy as np import pandas as pd @@ -103,6 +103,9 @@ def read_nav(ship, pathlist, sampling=1, talker=None, ship_function=None, progre 'R/V %s not yet supported for nav read; must supply read function' % ship) return -999 + if type(timest) is int and timest == -999: + return timest # eg looking for a talker that is not in a file + # posix, seconds, for interpolation sec_time = np.array([d.timestamp() for d in timest]) _, idx = np.unique(sec_time, return_index=True) @@ -267,6 +270,8 @@ def _navdate_Ride(allnav, talker): inav = [talker in s for s in allnav] # find lines of file with this talker subnav = allnav[inav] # select only those lines N = len(subnav) + if N == 0: + return -999 # array for timestamps, as datetime objects timest = np.empty(N, dtype=datetime) @@ -274,7 +279,7 @@ def _navdate_Ride(allnav, talker): if talker == 'INGGA': # on Ride, uses posix timestamps date = re.findall(r'(\d+(\.\d*)?) \$%s' % talker, subnav[i])[0] timest[i] = datetime.fromtimestamp( - float(date[0]), tzinfo=timezone.utc) + float(date[0]), UTC) elif talker == 'GPGGA': # includes time only with date, unlike other GPGGAs date = re.findall( r'(\d{4})\-(\d{2})\-(\d{2})T(\d{2}):(\d{2}):(\d{2})\.(\d.*?)Z', subnav[i])[0] From 0983a1e4f2273c0bdfb5be06120649049f99c686 Mon Sep 17 00:00:00 2001 From: Hannah Mark Date: Fri, 1 Nov 2024 14:02:50 -0400 Subject: [PATCH 4/6] add a bunch more tests for better coverage --- shipgrav/tests/ex_files/AT01_bgm.BGM | 4 + shipgrav/tests/ex_files/AT01_nav.gps | 8 ++ shipgrav/tests/ex_files/MGL2003_bgm.y2020d244 | 4 + shipgrav/tests/ex_files/MGL2003_nav.y2020d244 | 12 +++ shipgrav/tests/ex_files/NBP_2301_nav.d013 | 22 +++++ shipgrav/tests/ex_files/RR2212_bgm.txt | 6 ++ shipgrav/tests/ex_files/RR2212_nav.txt | 19 ++++ shipgrav/tests/ex_files/SR2302_nav.raw | 36 ++++++++ shipgrav/tests/ex_files/TN400_dgs_proc.Raw | 2 + shipgrav/tests/ex_files/TN400_dgs_raw.Raw | Bin 0 -> 311 bytes shipgrav/tests/test_grav_nodata.py | 19 +++- shipgrav/tests/test_io.py | 87 ++++++++++++++++-- 12 files changed, 208 insertions(+), 11 deletions(-) create mode 100644 shipgrav/tests/ex_files/AT01_bgm.BGM create mode 100644 shipgrav/tests/ex_files/AT01_nav.gps create mode 100644 shipgrav/tests/ex_files/MGL2003_bgm.y2020d244 create mode 100644 shipgrav/tests/ex_files/MGL2003_nav.y2020d244 create mode 100644 shipgrav/tests/ex_files/NBP_2301_nav.d013 create mode 100644 shipgrav/tests/ex_files/RR2212_bgm.txt create mode 100644 shipgrav/tests/ex_files/RR2212_nav.txt create mode 100644 shipgrav/tests/ex_files/SR2302_nav.raw create mode 100644 shipgrav/tests/ex_files/TN400_dgs_proc.Raw create mode 100644 shipgrav/tests/ex_files/TN400_dgs_raw.Raw diff --git a/shipgrav/tests/ex_files/AT01_bgm.BGM b/shipgrav/tests/ex_files/AT01_bgm.BGM new file mode 100644 index 0000000..d5bb7ac --- /dev/null +++ b/shipgrav/tests/ex_files/AT01_bgm.BGM @@ -0,0 +1,4 @@ +BGM 2022/07/01 12:17:33.492 BGM3 04:024963 00 +BGM 2022/07/01 12:17:34.492 BGM3 04:024982 00 +BGM 2022/07/01 12:17:35.492 BGM3 04:024992 00 +BGM 2022/07/01 12:17:36.492 BGM3 04:024983 00 diff --git a/shipgrav/tests/ex_files/AT01_nav.gps b/shipgrav/tests/ex_files/AT01_nav.gps new file mode 100644 index 0000000..9ab96a7 --- /dev/null +++ b/shipgrav/tests/ex_files/AT01_nav.gps @@ -0,0 +1,8 @@ +NAV 2022/07/01 12:17:33.233 GPS $GPZDA,121733.00,01,07,2022,00,00*67 +NAV 2022/07/01 12:17:33.433 GPS $GPRMC,121733.00,A,4131.403226,N,07040.311159,W,0.01,218.7,010722,,,D*7C +NAV 2022/07/01 12:17:33.433 GPS $GPVTG,218.7,T,,M,0.01,N,0.02,K,D*07 +NAV 2022/07/01 12:17:33.537 GPS $GPGGA,121733.00,4131.403226,N,07040.311159,W,2,12,0.9,25.169,M,-30.062,M,4.0,0402*46 +NAV 2022/07/01 12:17:34.233 GPS $GPZDA,121734.00,01,07,2022,00,00*60 +NAV 2022/07/01 12:17:34.433 GPS $GPRMC,121734.00,A,4131.403225,N,07040.311162,W,0.00,206.6,010722,,,D*7F +NAV 2022/07/01 12:17:34.433 GPS $GPVTG,206.6,T,,M,0.00,N,0.01,K,D*0B +NAV 2022/07/01 12:17:34.537 GPS $GPGGA,121734.00,4131.403225,N,07040.311162,W,2,12,0.9,25.176,M,-30.062,M,5.0,0402*45 diff --git a/shipgrav/tests/ex_files/MGL2003_bgm.y2020d244 b/shipgrav/tests/ex_files/MGL2003_bgm.y2020d244 new file mode 100644 index 0000000..0e6b443 --- /dev/null +++ b/shipgrav/tests/ex_files/MGL2003_bgm.y2020d244 @@ -0,0 +1,4 @@ +vc01 2020:244:00:00:00.3244 04:025229 00 +vc01 2020:244:00:00:01.3244 04:025247 00 +vc01 2020:244:00:00:02.3235 04:025404 00 +vc01 2020:244:00:00:03.3248 04:025535 00 diff --git a/shipgrav/tests/ex_files/MGL2003_nav.y2020d244 b/shipgrav/tests/ex_files/MGL2003_nav.y2020d244 new file mode 100644 index 0000000..57de610 --- /dev/null +++ b/shipgrav/tests/ex_files/MGL2003_nav.y2020d244 @@ -0,0 +1,12 @@ +seapath 2020:244:00:00:00.6212 $INGGA,000000.37,5420.890903,N,13237.257959,W,2,09,1.2,1.62,M,-7.75,M,4.2,0291*55 +seapath 2020:244:00:00:00.8547 $INGLL,5420.890903,N,13237.257959,W,000000.37,A,D*63 +seapath 2020:244:00:00:00.8549 $INVTG,77.26,T,59.62,M,9.8,N,18.1,K,D*03 +seapath 2020:244:00:00:00.8550 $INHDT,75.05,T*22 +seapath 2020:244:00:00:00.8551 $PSXN,20,0,0,0,0*3B +seapath 2020:244:00:00:00.9253 $PSXN,23,1.06,0.15,75.05,0.04*08 +seapath 2020:244:00:00:01.6218 $INGGA,000001.37,5420.891483,N,13237.253415,W,2,09,1.2,1.60,M,-7.75,M,5.2,0291*52 +seapath 2020:244:00:00:01.8561 $INGLL,5420.891483,N,13237.253415,W,000001.37,A,D*67 +seapath 2020:244:00:00:01.8562 $INVTG,77.89,T,60.26,M,9.8,N,18.2,K,D*0F +seapath 2020:244:00:00:01.8563 $INHDT,74.82,T*2C +seapath 2020:244:00:00:01.8564 $PSXN,20,0,0,0,0*3B +seapath 2020:244:00:00:01.9253 $PSXN,23,1.09,0.10,74.82,0.05*0D diff --git a/shipgrav/tests/ex_files/NBP_2301_nav.d013 b/shipgrav/tests/ex_files/NBP_2301_nav.d013 new file mode 100644 index 0000000..6d82d16 --- /dev/null +++ b/shipgrav/tests/ex_files/NBP_2301_nav.d013 @@ -0,0 +1,22 @@ +23+013:00:00:00.113 $GPVTG,5.71,T,,M,5.2,N,9.6,K,A*36 +23+013:00:00:00.115 $GPRMC,000000.81,A,7520.156405,S,17945.635852,W,5.2,5.71,130123,,,A*6C +23+013:00:00:00.213 $GPHDT,4.62,T*05 +23+013:00:00:00.213 $PSXN,20,0,0,0,0*3B +23+013:00:00:00.213 $PSXN,22,1.32,0.09*30 +23+013:00:00:00.213 $PSXN,23,-0.67,0.42,4.62,0.10*13 +23+013:00:00:00.996 $GPZDA,000001.81,13,01,2023,,*6E +23+013:00:00:00.996 $GPGGA,000001.81,7520.154976,S,17945.635106,W,1,12,0.5,-1.36,M,-56.73,M,,*6F +23+013:00:00:01.113 $GPVTG,6.29,T,,M,5.1,N,9.5,K,A*38 +23+013:00:00:01.113 $GPRMC,000001.81,A,7520.154976,S,17945.635106,W,5.1,6.29,130123,,,A*63 +23+013:00:00:01.213 $GPHDT,4.74,T*02 +23+013:00:00:01.213 $PSXN,20,0,0,0,0*3B +23+013:00:00:01.213 $PSXN,22,1.32,0.09*30 +23+013:00:00:01.213 $PSXN,23,-0.46,0.32,4.74,0.06*17 +23+013:00:00:02.012 $GPZDA,000002.81,13,01,2023,,*6D +23+013:00:00:02.012 $GPGGA,000002.81,7520.153540,S,17945.635196,W,1,12,0.5,-1.29,M,-56.73,M,,*65 +23+013:00:00:02.142 $GPVTG,2.95,T,,M,5.1,N,9.5,K,A*3B +23+013:00:00:02.142 $GPRMC,000002.81,A,7520.153540,S,17945.635196,W,5.1,2.95,130123,,,A*64 +23+013:00:00:02.213 $GPHDT,4.81,T*08 +23+013:00:00:02.213 $PSXN,20,0,0,0,0*3B +23+013:00:00:02.213 $PSXN,22,1.32,0.09*30 +23+013:00:00:02.214 $PSXN,23,-0.61,0.22,4.81,0.01*1E diff --git a/shipgrav/tests/ex_files/RR2212_bgm.txt b/shipgrav/tests/ex_files/RR2212_bgm.txt new file mode 100644 index 0000000..1bbfd17 --- /dev/null +++ b/shipgrav/tests/ex_files/RR2212_bgm.txt @@ -0,0 +1,6 @@ +2022-11-07T17:13:45.329611Z 04:024882 00 +2022-11-07T17:13:46.336966Z 04:024881 00 +2022-11-07T17:13:47.329375Z 04:024871 00 +2022-11-07T17:13:48.329063Z 04:024860 00 +2022-11-07T17:13:49.328981Z 04:024850 00 +2022-11-07T17:13:50.328786Z 04:024840 00 diff --git a/shipgrav/tests/ex_files/RR2212_nav.txt b/shipgrav/tests/ex_files/RR2212_nav.txt new file mode 100644 index 0000000..3cc519f --- /dev/null +++ b/shipgrav/tests/ex_files/RR2212_nav.txt @@ -0,0 +1,19 @@ +2022-11-05T00:00:00.002010Z $GPZDA,000000.00,05,11,2022,,*61 +$GPGGA,000000.00,3345.095825,N,11923.337492,W,2,09,0.9,-3.95,M,-33.51,M,2.0,0135*5F +$GPGLL,3345.095825,N,11923.337492,W,000000.00,A,D*7C +$GPVTG,200.16,T,,M,11.0,N,20.4,K,D*3B +$GPVBW,,,V,10.99,-0.08,A,,V,,V*57 +$GPRMC,000000.00,A,3345.095825,N,11923.337492,W,11.0,200.16,051122,,,D*4B +$GPHDT,200.59,T*0B +$PSXN,20,0,0,0,0*3B +$PSXN,23,-1.34,0.12,200.59,-0.07*34 + +2022-11-05T00:00:00.202094Z $GPZDA,000000.20,05,11,2022,,*63 +$GPGGA,000000.20,3345.095252,N,11923.337748,W,2,09,0.9,-4.00,M,-33.51,M,2.0,0135*58 +$GPGLL,3345.095252,N,11923.337748,W,000000.20,A,D*70 +$GPVTG,200.60,T,,M,11.0,N,20.4,K,D*3A +$GPVBW,,,V,11.00,-0.02,A,,V,,V*5C +$GPRMC,000000.20,A,3345.095252,N,11923.337748,W,11.0,200.60,051122,,,D*46 +$GPHDT,200.70,T*00 +$PSXN,20,0,0,0,0*3B +$PSXN,23,-1.32,0.06,200.70,-0.03*38 diff --git a/shipgrav/tests/ex_files/SR2302_nav.raw b/shipgrav/tests/ex_files/SR2302_nav.raw new file mode 100644 index 0000000..92b37c2 --- /dev/null +++ b/shipgrav/tests/ex_files/SR2302_nav.raw @@ -0,0 +1,36 @@ +1674228947.582 $INZDA,153547.44,20,01,2023,,*77 +1674228947.583 $INGGA,153547.44,3242.434865,N,11714.203305,W,2,09,0.8,-4.31,M,-33.32,M,2.0,0135*4B +1674228947.583 $INGLL,3242.434865,N,11714.203305,W,153547.44,A,D*65 +1674228947.583 $INVTG,112.40,T,,M,0.0,N,0.1,K,D*2F +1674228947.583 $INRMC,153547.44,A,3242.434865,N,11714.203305,W,0.0,112.40,200123,,,D*66 +1674228947.603 $GPGSA,M,3,21,31,09,16,03,01,26,04,22,,,,1.7,0.8,1.5*39 +1674228947.633 $INHDT,179.39,T*10 +1674228947.643 $PSXN,20,0,0,0,0*3B +1674228947.653 $PSXN,23,-0.33,0.42,179.39,0.01*17 +1674228947.673 $PSXN,26,0,44.5000,0.7800,-0.9000,NRP*6D +1674228947.693 $PSXN,26,1,44.5000,0.7800,-0.9000,Monitoring Point #1*44 +1674228947.723 $PSXN,26,2,44.5000,0.7800,-0.9000,Monitoring Point #2*1F +1674228947.754 $PSXN,26,3,44.5000,0.7800,-0.9000,Monitoring Point #3*33 +1674228947.784 $PSXN,26,4,44.5000,0.7800,-0.9000,Monitoring Point #4*68 +1674228947.814 $PSXN,26,5,44.5000,0.7800,-0.9000,Monitoring Point #5*3D +1674228947.844 $PSXN,26,6,44.5000,0.7800,-0.9000,Monitoring Point #6*11 +1674228947.874 $PSXN,26,7,44.5000,0.7800,-0.9000,Monitoring Point #7*4A +1674228947.904 $PSXN,26,8,44.5000,0.7800,-0.9000,Monitoring Point #8*64 +1674228948.444 $INZDA,153548.44,20,01,2023,,*78 +1674228948.474 $INGGA,153548.44,3242.434850,N,11714.203293,W,2,09,0.8,-4.32,M,-33.32,M,3.0,0135*4E +1674228948.514 $INGLL,3242.434850,N,11714.203293,W,153548.44,A,D*62 +1674228948.544 $INVTG,117.77,T,,M,0.0,N,0.1,K,D*2E +1674228948.564 $INRMC,153548.44,A,3242.434850,N,11714.203293,W,0.0,117.77,200123,,,D*60 +1674228948.604 $GPGSA,M,3,21,31,09,16,03,01,26,04,22,,,,1.7,0.8,1.5*39 +1674228948.634 $INHDT,179.38,T*11 +1674228948.644 $PSXN,20,0,0,0,0*3B +1674228948.654 $PSXN,23,-0.27,0.41,179.38,0.01*10 +1674228948.674 $PSXN,26,0,44.5000,0.7800,-0.9000,NRP*6D +1674228948.694 $PSXN,26,1,44.5000,0.7800,-0.9000,Monitoring Point #1*44 +1674228948.724 $PSXN,26,2,44.5000,0.7800,-0.9000,Monitoring Point #2*1F +1674228948.754 $PSXN,26,3,44.5000,0.7800,-0.9000,Monitoring Point #3*33 +1674228948.784 $PSXN,26,4,44.5000,0.7800,-0.9000,Monitoring Point #4*68 +1674228948.814 $PSXN,26,5,44.5000,0.7800,-0.9000,Monitoring Point #5*3D +1674228948.844 $PSXN,26,6,44.5000,0.7800,-0.9000,Monitoring Point #6*11 +1674228948.874 $PSXN,26,7,44.5000,0.7800,-0.9000,Monitoring Point #7*4A +1674228948.904 $PSXN,26,8,44.5000,0.7800,-0.9000,Monitoring Point #8*64 diff --git a/shipgrav/tests/ex_files/TN400_dgs_proc.Raw b/shipgrav/tests/ex_files/TN400_dgs_proc.Raw new file mode 100644 index 0000000..ed5a835 --- /dev/null +++ b/shipgrav/tests/ex_files/TN400_dgs_proc.Raw @@ -0,0 +1,2 @@ +03/12/2022,16:04:07.033,9984.869108166342,9995.951860660536, 0.031769279452, 0.049233442454, -0.894949104184, 62.394784400000, 16730, 4086876, 30.259833080000, 50.442681000000, 0.000020000000, 0.035710000000, 0.000070000000, -0.000010000000, 0.000000000000, 0.000000000000, 0.000000000000, 0.000000000000, 0.044215194148,2022,03,12,16,04,19.80,5370472 +03/12/2022,16:04:08.017,9984.870433770533,9977.883336291710, 0.030815605022, 0.037431721381, -0.894945527905, 62.394784400000, 16730, 4086005, 30.260701895000, 50.447645000000, 0.000020000000, 0.032780000000, 0.000020000000, -0.000010000000, 0.000000000000, 0.000000000000, 0.000000000000, 0.000000000000, 0.044061782566,2022,03,12,16,04,20.80,5370473 diff --git a/shipgrav/tests/ex_files/TN400_dgs_raw.Raw b/shipgrav/tests/ex_files/TN400_dgs_raw.Raw new file mode 100644 index 0000000000000000000000000000000000000000..0c57ded2371c5180b92cf93aebb95b212e102be8 GIT binary patch literal 311 zcmZwCI}XA?3oP%-_#-3e!Nyh=`=yU&DunAERB5wSVB|lR)^PAHh7BkDtm_h~D z$IG6Jrm&?dS<*2!N0rb#P}s~!IPa)tJrkZWzJW datetime -> posix - nav = sgi.read_nav('Thompson', f'{self.ex_files}'+os.sep+'TN400_nav.Raw') + def test_read_nav_Thompson(self): + nav = sgi.read_nav('Thompson', f'{self.ex_files}'+os.sep+'TN400_nav.Raw', progressbar=False) self.assertEqual(nav.iloc[0].time_sec, 1647129603) self.assertTrue(nav.iloc[0].lon + 118.6524 < 0.001) + def test_read_nav_Atlantis(self): + nav = sgi.read_nav('Atlantis', f'{self.ex_files}'+os.sep+'AT01_nav.gps', progressbar=False) + self.assertEqual(nav.iloc[0].time_sec, 1656677853) + self.assertTrue(nav.iloc[0].lon + 70.67185265 < 0.001) + + def test_read_nav_Langseth(self): + nav = sgi.read_nav('Langseth', f'{self.ex_files}'+os.sep+'MGL2003_nav.y2020d244', progressbar=False) + self.assertEqual(nav.iloc[0].time_sec, 1598832000.6212) + self.assertTrue(nav.iloc[0].lon + 132.620965893 < 0.001) + + def test_read_nav_Revelle(self): + nav = sgi.read_nav('Revelle', f'{self.ex_files}'+os.sep+'RR2212_nav.txt', progressbar=False) + self.assertEqual(nav.iloc[0].time_sec, 1667606400) + self.assertTrue(nav.iloc[0].lon + 119.3889582 < 0.001) + + def test_read_nav_Ride(self): + nav = sgi.read_nav('Ride', f'{self.ex_files}'+os.sep+'SR2302_nav.raw',talker='INGGA', progressbar=False) + self.assertEqual(nav.iloc[0].time_sec, 1674228947.583) + self.assertTrue(nav.iloc[0].lon + 117.23672175 < 0.001) + + def test_read_nav_NBP(self): + nav = sgi.read_nav('NBP', f'{self.ex_files}'+os.sep+'NBP_2301_nav.d013', progressbar=False) + self.assertEqual(nav.iloc[0].time_sec, 1673568001.81) + self.assertTrue(nav.iloc[0].lon + 179.7605851 < 0.001) + + def test_read_nav_nope(self): + nav = sgi.read_nav('Titanic', f'{self.ex_files}'+os.sep+'TN400_nav.Raw', progressbar=False) + self.assertEqual(nav, -999) + def test_read_bgm_rgs(self): - bgm = sgi.read_bgm_rgs(f'{self.ex_files}'+os.sep+'AT05_01_bgm.RGS', 'Atlantis') + bgm = sgi.read_bgm_rgs(f'{self.ex_files}'+os.sep+'AT05_01_bgm.RGS', 'Atlantis', progressbar=False) self.assertEqual(bgm.iloc[0]['date_time'].timestamp(), 1656633600.445) self.assertEqual(bgm.iloc[0]['grav'], 980329.272) - def test_read_bgm_raw(self): - bgm = sgi.read_bgm_raw(f'{self.ex_files}'+os.sep+'TN400_bgm.Raw', 'Thompson') + def test_read_bgm_raw_Atlantis(self): + bgm = sgi.read_bgm_raw(f'{self.ex_files}'+os.sep+'AT01_bgm.BGM', 'Atlantis', progressbar=False) + self.assertEqual(bgm.iloc[0]['date_time'].timestamp(), 1656677853.492) + self.assertEqual(bgm.iloc[0]['counts'], 24963) + self.assertEqual(bgm.iloc[0]['rgrav'], 124666.983189576) + + def test_read_bgm_raw_Thompson(self): + bgm = sgi.read_bgm_raw(f'{self.ex_files}'+os.sep+'TN400_bgm.Raw', 'Thompson', progressbar=False) self.assertEqual(bgm.iloc[0]['date_time'].timestamp(), 1647129602.449) self.assertEqual(bgm.iloc[0]['counts'], 25529) self.assertEqual(bgm.iloc[0]['rgrav'], 127730.60800402702) - def test_read_dgs_dat(self): - dgs = sgi.read_dgs_laptop(f'{self.ex_files}'+os.sep+'DGStest_laptop.dat', 'DGStest') + def test_read_bgm_raw_Langseth(self): + bgm = sgi.read_bgm_raw(f'{self.ex_files}'+os.sep+'MGL2003_bgm.y2020d244', 'Langseth', progressbar=False) + self.assertEqual(bgm.iloc[0]['date_time'].timestamp(), 1598832000.3244) + self.assertEqual(bgm.iloc[0]['counts'], 25229) + self.assertEqual(bgm.iloc[0]['rgrav'], 126145.0) + + def test_read_bgm_raw_Revelle(self): + bgm = sgi.read_bgm_raw(f'{self.ex_files}'+os.sep+'RR2212_bgm.txt', 'Revelle', progressbar=False) + self.assertEqual(bgm.iloc[0]['date_time'].timestamp(), 1667841225.329611) + self.assertEqual(bgm.iloc[0]['counts'], 24882) + self.assertEqual(bgm.iloc[0]['rgrav'], 124405.2114591) + + def test_read_bgm_raw_nope(self): + bgm = sgi.read_bgm_raw(f'{self.ex_files}'+os.sep+'TN400_bgm.Raw', 'Boaty McBoatface', progressbar=False) + self.assertEqual(bgm, -999) + + def test_read_dgs_dat_general(self): + dgs = sgi.read_dgs_laptop(f'{self.ex_files}'+os.sep+'DGStest_laptop.dat', 'DGStest', progressbar=False) self.assertEqual(dgs.iloc[0]['date_time'].timestamp(), 1562803200.0) self.assertEqual(dgs.iloc[0]['ve'], 0.81098) self.assertTrue(dgs.iloc[0]['rgrav'] - 12295.691114 < 0.0001) - def test_read_dgs_raw(self): - dgs = sgi.read_dgs_raw(f'{self.ex_files}'+os.sep+'SR2312_dgs_raw.txt', 'Ride') + def test_read_dgs_dat_Thompson(self): + dgs = sgi.read_dgs_laptop(f'{self.ex_files}'+os.sep+'TN400_dgs_proc.Raw', 'Thompson', progressbar=False) + self.assertEqual(dgs.iloc[0]['date_time'].timestamp(), 1647101047.033) + self.assertEqual(dgs.iloc[0]['ve'], 2e-5) + self.assertTrue(dgs.iloc[0]['rgrav'] - 9995.95186 < 0.0001) + + def test_read_dgs_dat_nope(self): + dgs = sgi.read_dgs_laptop(f'{self.ex_files}'+os.sep+'DGStest_laptop.dat', 'Katama', progressbar=False) + self.assertEqual(dgs, -999) + + def test_read_dgs_raw_general(self): + dgs = sgi.read_dgs_raw(f'{self.ex_files}'+os.sep+'SR2312_dgs_raw.txt', 'Ride', progressbar=False) self.assertEqual( dgs.iloc[0]['date_time'].timestamp(), 1686873600.857719) self.assertEqual(dgs.iloc[0]['Gravity'], -218747) self.assertTrue(dgs.iloc[0]['vcc'] - 76.8771 < 0.0001) + def test_read_dgs_raw_Thompson(self): + dgs = sgi.read_dgs_raw(f'{self.ex_files}'+os.sep+'TN400_dgs_raw.Raw', 'Thompson', progressbar=False) + self.assertEqual( + dgs.iloc[0]['date_time'].timestamp(), 1647101046.634) + self.assertEqual(dgs.iloc[0]['Gravity'], -82) + self.assertTrue(dgs.iloc[0]['vcc'] - 0.0357100 < 0.0001) + def test_read_mru(self): mru, cols = sgi.read_other_stuff( f'{self.ex_files}'+os.sep+'IXBlue.yaml', f'{self.ex_files}'+os.sep+'SR2312_mru.txt', 'PASHR') From 0a9ec7d30b8e2a0a06ef62afe05aed30c88a359a Mon Sep 17 00:00:00 2001 From: Hannah Mark Date: Fri, 1 Nov 2024 14:08:53 -0400 Subject: [PATCH 5/6] utc name for python<3.13 --- shipgrav/io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shipgrav/io.py b/shipgrav/io.py index 428ba30..92a5d6f 100644 --- a/shipgrav/io.py +++ b/shipgrav/io.py @@ -1,7 +1,7 @@ import mmap import os import re -from datetime import datetime, timezone, UTC +from datetime import datetime, timezone import numpy as np import pandas as pd @@ -279,7 +279,7 @@ def _navdate_Ride(allnav, talker): if talker == 'INGGA': # on Ride, uses posix timestamps date = re.findall(r'(\d+(\.\d*)?) \$%s' % talker, subnav[i])[0] timest[i] = datetime.fromtimestamp( - float(date[0]), UTC) + float(date[0]), timezone.utc) elif talker == 'GPGGA': # includes time only with date, unlike other GPGGAs date = re.findall( r'(\d{4})\-(\d{2})\-(\d{2})T(\d{2}):(\d{2}):(\d{2})\.(\d.*?)Z', subnav[i])[0] From cc38580f7e1e0bf60ba9bf14152cc0f7035f3f5a Mon Sep 17 00:00:00 2001 From: Hannah Mark Date: Fri, 1 Nov 2024 14:14:35 -0400 Subject: [PATCH 6/6] better negation for progress bar flag --- shipgrav/io.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/shipgrav/io.py b/shipgrav/io.py index 92a5d6f..e364fb6 100644 --- a/shipgrav/io.py +++ b/shipgrav/io.py @@ -73,7 +73,7 @@ def read_nav(ship, pathlist, sampling=1, talker=None, ship_function=None, progre lonlon = np.array([]) latlat = np.array([]) - for fpath in tqdm(pathlist,desc='reading nav',disable=~progressbar): # loop nav files (may be a lot of them) + for fpath in tqdm(pathlist,desc='reading nav',disable=not progressbar): # loop nav files (may be a lot of them) with open(fpath, 'r') as f: allnav = np.array(f.readlines()) # read the entire file @@ -391,7 +391,7 @@ def read_bgm_rgs(fp, ship, progressbar=True): fp = [fp,] # make a list if only one path is given dats = [] - for path in tqdm(fp, desc='reading RGS files', disable=~progressbar): + for path in tqdm(fp, desc='reading RGS files', disable=not progressbar): dat = pd.read_csv(path, delimiter=' ', names=['date', 'time', 'grav', 'lat', 'lon'], usecols=(1, 2, 3, 11, 12)) dat['date_time'] = pd.to_datetime(dat.pop('date')+' '+dat.pop('time'),utc=True) @@ -441,7 +441,7 @@ def read_bgm_raw(fp, ship, scale=None, ship_function=None, progressbar=True): if type(fp) is str: fp = [fp,] # make a list if only one path is given dats = [] - for path in tqdm(fp,desc='reading BGM files',disable=~progressbar): + for path in tqdm(fp,desc='reading BGM files',disable=not progressbar): if ship_function != None: dat = ship_function(path) else: @@ -549,7 +549,7 @@ def read_dgs_laptop(fp, ship, ship_function=None, progressbar=True): fp = [fp,] # listify dats = [] - for path in tqdm(fp,desc='reading DGS files',disable=~progressbar): + for path in tqdm(fp,desc='reading DGS files',disable=not progressbar): if ship_function is not None: dat = ship_function(path) else: @@ -609,7 +609,7 @@ def read_dgs_raw(fp, ship, scale_ccp=True, progressbar=True): if type(fp) is str: fp = [fp,] # listify dats = [] - for path in tqdm(fp,desc='reading DGS files',disable=~progressbar): + for path in tqdm(fp,desc='reading DGS files',disable=not progressbar): if ship == 'Thompson': # always with the special file formats dat = _dgs_raw_Thompson(path) else: # there might be exceptions besides Thompson but I don't know about them yet