Increase coverage

shipgrav/io.py

@@ -17,7 +17,7 @@
 ########################################################################
 
 
-def read_nav(ship, pathlist, sampling=1, talker=None, ship_function=None):
+def read_nav(ship, pathlist, sampling=1, talker=None, ship_function=None, progressbar=True):
     """ Read navigation strings from .GPS (or similar) files.
 
     Ships have different formats and use different talkers for preferred
@@ -38,6 +38,8 @@ def read_nav(ship, pathlist, sampling=1, talker=None, ship_function=None):
         This function should return arrays of lon, lat, and timestamps.
         Look at _navcoords() and navdate_Atlantis() (and similar functions) for examples.
     :type ship_function: function, optional
+    :param progressbar: display progress bar while list of files is read
+    :type progressbar: bool
 
     :returns: (*pd.DataFrame*) time series of geographic coordinates and timestamps
     """
@@ -71,7 +73,7 @@ def read_nav(ship, pathlist, sampling=1, talker=None, ship_function=None):
     lonlon = np.array([])
     latlat = np.array([])
 
-    for fpath in tqdm(pathlist,desc='reading nav'):  # loop nav files (may be a lot of them)
+    for fpath in tqdm(pathlist,desc='reading nav',disable=not progressbar):  # loop nav files (may be a lot of them)
         with open(fpath, 'r') as f:
             allnav = np.array(f.readlines())  # read the entire file
 
@@ -101,6 +103,9 @@ def read_nav(ship, pathlist, sampling=1, talker=None, ship_function=None):
                     'R/V %s not yet supported for nav read; must supply read function' % ship)
                 return -999
 
+        if type(timest) is int and timest == -999:
+            return timest  # eg looking for a talker that is not in a file
+
         # posix, seconds, for interpolation
         sec_time = np.array([d.timestamp() for d in timest])
         _, idx = np.unique(sec_time, return_index=True)
@@ -265,14 +270,16 @@ def _navdate_Ride(allnav, talker):
     inav = [talker in s for s in allnav]  # find lines of file with this talker
     subnav = allnav[inav]  # select only those lines
     N = len(subnav)
+    if N == 0:
+        return -999
     # array for timestamps, as datetime objects
     timest = np.empty(N, dtype=datetime)
 
     for i in range(N):
         if talker == 'INGGA':  # on Ride, uses posix timestamps
             date = re.findall(r'(\d+(\.\d*)?) \$%s' % talker, subnav[i])[0]
             timest[i] = datetime.fromtimestamp(
-                float(date[0]), tzinfo=timezone.utc)
+                float(date[0]), timezone.utc)
         elif talker == 'GPGGA':  # includes time only with date, unlike other GPGGAs
             date = re.findall(
                 r'(\d{4})\-(\d{2})\-(\d{2})T(\d{2}):(\d{2}):(\d{2})\.(\d.*?)Z', subnav[i])[0]
@@ -360,7 +367,7 @@ def _clean_180cross(gps_nav):
 ########################################################################
 
 
-def read_bgm_rgs(fp, ship):
+def read_bgm_rgs(fp, ship, progressbar=True):
     """Read BGM gravity from RGS files.
 
     RGS is supposedly a standard format; is consistent between Atlantis 
@@ -370,6 +377,8 @@ def read_bgm_rgs(fp, ship):
     :type fp: string, or list of strings
     :param ship: ship name
     :type ship: string
+    :param progressbar: display progress bar while list of files is read
+    :type progressbar: bool
 
     :returns: (*pd.DataFrame*) timestamps, raw gravity, and geographic coordinates
     """
@@ -382,7 +391,7 @@ def read_bgm_rgs(fp, ship):
         fp = [fp,]  # make a list if only one path is given
 
     dats = []
-    for path in fp:
+    for path in tqdm(fp, desc='reading RGS files', disable=not progressbar):
         dat = pd.read_csv(path, delimiter=' ', names=['date', 'time', 'grav', 'lat', 'lon'],
                           usecols=(1, 2, 3, 11, 12))
         dat['date_time'] = pd.to_datetime(dat.pop('date')+' '+dat.pop('time'),utc=True)
@@ -391,7 +400,7 @@ def read_bgm_rgs(fp, ship):
     return pd.concat(dats, ignore_index=True)
 
 
-def read_bgm_raw(fp, ship, scale=None, ship_function=None):
+def read_bgm_raw(fp, ship, scale=None, ship_function=None, progressbar=True):
     """Read BGM gravity from raw (serial) files (not RGS).
 
     This function uses scale factors determined for specific BGM meters
@@ -408,6 +417,8 @@ def read_bgm_raw(fp, ship, scale=None, ship_function=None):
         The function should return a pandas.DataFrame with timestamps and counts.
         Look at _bgmserial_Atlantis() and similar functions for examples.
     :type ship_function: function, optional
+    :param progressbar: display progress bar while list of files is read
+    :type progressbar: bool
 
     :return: (*pd.DataFrame*) timestamps and calibrated raw gravity values
     """
@@ -430,7 +441,7 @@ def read_bgm_raw(fp, ship, scale=None, ship_function=None):
     if type(fp) is str:
         fp = [fp,]  # make a list if only one path is given
     dats = []
-    for path in fp:
+    for path in tqdm(fp,desc='reading BGM files',disable=not progressbar):
         if ship_function != None:
             dat = ship_function(path)
         else:
@@ -477,8 +488,6 @@ def _bgmserial_Revelle(path):
     def count(x): return (int(x.split(':')[-1]))
     dat = pd.read_csv(path, delimiter=' ', names=['date_time', 'counts'], usecols=(0, 1),
                       parse_dates=[0], converters={'counts': count})
-    ndt = [e.tz_localize(timezone.utc) for e in dat['date_time']]
-    dat['date_time'] = ndt
     return dat
 
 def _bgmserial_Langseth(path):
@@ -520,7 +529,7 @@ def _despike_bgm_serial(dat, thresh=8000):
 ########################################################################
 
 
-def read_dgs_laptop(fp, ship, ship_function=None):
+def read_dgs_laptop(fp, ship, ship_function=None, progressbar=True):
     """Read DGS 'laptop' file(s), usually written as .dat files.
 
     :param fp: filepath(s)
@@ -531,14 +540,16 @@ def read_dgs_laptop(fp, ship, ship_function=None):
         The function should return a pandas.DataFrame. See _dgs_laptop_general()
         for an example.
     :type ship_function: function, optional
+    :param progressbar: display progress bar while list of files is read
+    :type progressbar: bool
 
     :return: *(pd.DataFrame)* DGS output time series
     """
     if type(fp) is str:
         fp = [fp,]  # listify
 
     dats = []
-    for path in tqdm(fp,desc='reading DGS files'):
+    for path in tqdm(fp,desc='reading DGS files',disable=not progressbar):
         if ship_function is not None:
             dat = ship_function(path)
         else:
@@ -577,7 +588,7 @@ def _dgs_laptop_Thompson(path):
     return dat
 
 
-def read_dgs_raw(fp, ship, scale_ccp=True):
+def read_dgs_raw(fp, ship, scale_ccp=True, progressbar=True):
     """Read raw (serial) output files from DGS AT1M.
 
     These will be in AD units mostly.
@@ -589,14 +600,16 @@ def read_dgs_raw(fp, ship, scale_ccp=True):
     :type fp: string or list of strings
     :param ship: ship name
     :type ship: string
+    :param progressbar: display progress bar while list of files is read
+    :type progressbar: bool
 
     :return: (*pd.DataFrame*) DGS output time series
     """
 
     if type(fp) is str:
         fp = [fp,]  # listify
     dats = []
-    for path in tqdm(fp,desc='reading DGS files'):
+    for path in tqdm(fp,desc='reading DGS files',disable=not progressbar):
         if ship == 'Thompson':  # always with the special file formats
             dat = _dgs_raw_Thompson(path)
         else:  # there might be exceptions besides Thompson but I don't know about them yet

shipgrav/tests/ex_files/AT01_bgm.BGM

@@ -0,0 +1,4 @@
+BGM 2022/07/01 12:17:33.492 BGM3 04:024963 00
+BGM 2022/07/01 12:17:34.492 BGM3 04:024982 00
+BGM 2022/07/01 12:17:35.492 BGM3 04:024992 00
+BGM 2022/07/01 12:17:36.492 BGM3 04:024983 00

shipgrav/tests/ex_files/AT01_nav.gps

@@ -0,0 +1,8 @@
+NAV 2022/07/01 12:17:33.233 GPS $GPZDA,121733.00,01,07,2022,00,00*67
+NAV 2022/07/01 12:17:33.433 GPS $GPRMC,121733.00,A,4131.403226,N,07040.311159,W,0.01,218.7,010722,,,D*7C
+NAV 2022/07/01 12:17:33.433 GPS $GPVTG,218.7,T,,M,0.01,N,0.02,K,D*07
+NAV 2022/07/01 12:17:33.537 GPS $GPGGA,121733.00,4131.403226,N,07040.311159,W,2,12,0.9,25.169,M,-30.062,M,4.0,0402*46
+NAV 2022/07/01 12:17:34.233 GPS $GPZDA,121734.00,01,07,2022,00,00*60
+NAV 2022/07/01 12:17:34.433 GPS $GPRMC,121734.00,A,4131.403225,N,07040.311162,W,0.00,206.6,010722,,,D*7F
+NAV 2022/07/01 12:17:34.433 GPS $GPVTG,206.6,T,,M,0.00,N,0.01,K,D*0B
+NAV 2022/07/01 12:17:34.537 GPS $GPGGA,121734.00,4131.403225,N,07040.311162,W,2,12,0.9,25.176,M,-30.062,M,5.0,0402*45

shipgrav/tests/ex_files/MGL2003_bgm.y2020d244

@@ -0,0 +1,4 @@
+vc01	2020:244:00:00:00.3244	04:025229 00
+vc01	2020:244:00:00:01.3244	04:025247 00
+vc01	2020:244:00:00:02.3235	04:025404 00
+vc01	2020:244:00:00:03.3248	04:025535 00

shipgrav/tests/ex_files/MGL2003_nav.y2020d244

@@ -0,0 +1,12 @@
+seapath	2020:244:00:00:00.6212	$INGGA,000000.37,5420.890903,N,13237.257959,W,2,09,1.2,1.62,M,-7.75,M,4.2,0291*55
+seapath	2020:244:00:00:00.8547	$INGLL,5420.890903,N,13237.257959,W,000000.37,A,D*63
+seapath	2020:244:00:00:00.8549	$INVTG,77.26,T,59.62,M,9.8,N,18.1,K,D*03
+seapath	2020:244:00:00:00.8550	$INHDT,75.05,T*22
+seapath	2020:244:00:00:00.8551	$PSXN,20,0,0,0,0*3B
+seapath	2020:244:00:00:00.9253	$PSXN,23,1.06,0.15,75.05,0.04*08
+seapath	2020:244:00:00:01.6218	$INGGA,000001.37,5420.891483,N,13237.253415,W,2,09,1.2,1.60,M,-7.75,M,5.2,0291*52
+seapath	2020:244:00:00:01.8561	$INGLL,5420.891483,N,13237.253415,W,000001.37,A,D*67
+seapath	2020:244:00:00:01.8562	$INVTG,77.89,T,60.26,M,9.8,N,18.2,K,D*0F
+seapath	2020:244:00:00:01.8563	$INHDT,74.82,T*2C
+seapath	2020:244:00:00:01.8564	$PSXN,20,0,0,0,0*3B
+seapath	2020:244:00:00:01.9253	$PSXN,23,1.09,0.10,74.82,0.05*0D

shipgrav/tests/ex_files/NBP_2301_nav.d013

@@ -0,0 +1,22 @@
+23+013:00:00:00.113 $GPVTG,5.71,T,,M,5.2,N,9.6,K,A*36
+23+013:00:00:00.115 $GPRMC,000000.81,A,7520.156405,S,17945.635852,W,5.2,5.71,130123,,,A*6C
+23+013:00:00:00.213 $GPHDT,4.62,T*05
+23+013:00:00:00.213 $PSXN,20,0,0,0,0*3B
+23+013:00:00:00.213 $PSXN,22,1.32,0.09*30
+23+013:00:00:00.213 $PSXN,23,-0.67,0.42,4.62,0.10*13
+23+013:00:00:00.996 $GPZDA,000001.81,13,01,2023,,*6E
+23+013:00:00:00.996 $GPGGA,000001.81,7520.154976,S,17945.635106,W,1,12,0.5,-1.36,M,-56.73,M,,*6F
+23+013:00:00:01.113 $GPVTG,6.29,T,,M,5.1,N,9.5,K,A*38
+23+013:00:00:01.113 $GPRMC,000001.81,A,7520.154976,S,17945.635106,W,5.1,6.29,130123,,,A*63
+23+013:00:00:01.213 $GPHDT,4.74,T*02
+23+013:00:00:01.213 $PSXN,20,0,0,0,0*3B
+23+013:00:00:01.213 $PSXN,22,1.32,0.09*30
+23+013:00:00:01.213 $PSXN,23,-0.46,0.32,4.74,0.06*17
+23+013:00:00:02.012 $GPZDA,000002.81,13,01,2023,,*6D
+23+013:00:00:02.012 $GPGGA,000002.81,7520.153540,S,17945.635196,W,1,12,0.5,-1.29,M,-56.73,M,,*65
+23+013:00:00:02.142 $GPVTG,2.95,T,,M,5.1,N,9.5,K,A*3B
+23+013:00:00:02.142 $GPRMC,000002.81,A,7520.153540,S,17945.635196,W,5.1,2.95,130123,,,A*64
+23+013:00:00:02.213 $GPHDT,4.81,T*08
+23+013:00:00:02.213 $PSXN,20,0,0,0,0*3B
+23+013:00:00:02.213 $PSXN,22,1.32,0.09*30
+23+013:00:00:02.214 $PSXN,23,-0.61,0.22,4.81,0.01*1E

shipgrav/tests/ex_files/RR2212_bgm.txt

@@ -0,0 +1,6 @@
+2022-11-07T17:13:45.329611Z 04:024882 00
+2022-11-07T17:13:46.336966Z 04:024881 00
+2022-11-07T17:13:47.329375Z 04:024871 00
+2022-11-07T17:13:48.329063Z 04:024860 00
+2022-11-07T17:13:49.328981Z 04:024850 00
+2022-11-07T17:13:50.328786Z 04:024840 00

shipgrav/tests/ex_files/RR2212_nav.txt

@@ -0,0 +1,19 @@
+2022-11-05T00:00:00.002010Z $GPZDA,000000.00,05,11,2022,,*61
+$GPGGA,000000.00,3345.095825,N,11923.337492,W,2,09,0.9,-3.95,M,-33.51,M,2.0,0135*5F
+$GPGLL,3345.095825,N,11923.337492,W,000000.00,A,D*7C
+$GPVTG,200.16,T,,M,11.0,N,20.4,K,D*3B
+$GPVBW,,,V,10.99,-0.08,A,,V,,V*57
+$GPRMC,000000.00,A,3345.095825,N,11923.337492,W,11.0,200.16,051122,,,D*4B
+$GPHDT,200.59,T*0B
+$PSXN,20,0,0,0,0*3B
+$PSXN,23,-1.34,0.12,200.59,-0.07*34
+
+2022-11-05T00:00:00.202094Z $GPZDA,000000.20,05,11,2022,,*63
+$GPGGA,000000.20,3345.095252,N,11923.337748,W,2,09,0.9,-4.00,M,-33.51,M,2.0,0135*58
+$GPGLL,3345.095252,N,11923.337748,W,000000.20,A,D*70
+$GPVTG,200.60,T,,M,11.0,N,20.4,K,D*3A
+$GPVBW,,,V,11.00,-0.02,A,,V,,V*5C
+$GPRMC,000000.20,A,3345.095252,N,11923.337748,W,11.0,200.60,051122,,,D*46
+$GPHDT,200.70,T*00
+$PSXN,20,0,0,0,0*3B
+$PSXN,23,-1.32,0.06,200.70,-0.03*38

shipgrav/tests/ex_files/SR2302_nav.raw

@@ -0,0 +1,36 @@
+1674228947.582 $INZDA,153547.44,20,01,2023,,*77
+1674228947.583 $INGGA,153547.44,3242.434865,N,11714.203305,W,2,09,0.8,-4.31,M,-33.32,M,2.0,0135*4B
+1674228947.583 $INGLL,3242.434865,N,11714.203305,W,153547.44,A,D*65
+1674228947.583 $INVTG,112.40,T,,M,0.0,N,0.1,K,D*2F
+1674228947.583 $INRMC,153547.44,A,3242.434865,N,11714.203305,W,0.0,112.40,200123,,,D*66
+1674228947.603 $GPGSA,M,3,21,31,09,16,03,01,26,04,22,,,,1.7,0.8,1.5*39
+1674228947.633 $INHDT,179.39,T*10
+1674228947.643 $PSXN,20,0,0,0,0*3B
+1674228947.653 $PSXN,23,-0.33,0.42,179.39,0.01*17
+1674228947.673 $PSXN,26,0,44.5000,0.7800,-0.9000,NRP*6D
+1674228947.693 $PSXN,26,1,44.5000,0.7800,-0.9000,Monitoring Point #1*44
+1674228947.723 $PSXN,26,2,44.5000,0.7800,-0.9000,Monitoring Point #2*1F
+1674228947.754 $PSXN,26,3,44.5000,0.7800,-0.9000,Monitoring Point #3*33
+1674228947.784 $PSXN,26,4,44.5000,0.7800,-0.9000,Monitoring Point #4*68
+1674228947.814 $PSXN,26,5,44.5000,0.7800,-0.9000,Monitoring Point #5*3D
+1674228947.844 $PSXN,26,6,44.5000,0.7800,-0.9000,Monitoring Point #6*11
+1674228947.874 $PSXN,26,7,44.5000,0.7800,-0.9000,Monitoring Point #7*4A
+1674228947.904 $PSXN,26,8,44.5000,0.7800,-0.9000,Monitoring Point #8*64
+1674228948.444 $INZDA,153548.44,20,01,2023,,*78
+1674228948.474 $INGGA,153548.44,3242.434850,N,11714.203293,W,2,09,0.8,-4.32,M,-33.32,M,3.0,0135*4E
+1674228948.514 $INGLL,3242.434850,N,11714.203293,W,153548.44,A,D*62
+1674228948.544 $INVTG,117.77,T,,M,0.0,N,0.1,K,D*2E
+1674228948.564 $INRMC,153548.44,A,3242.434850,N,11714.203293,W,0.0,117.77,200123,,,D*60
+1674228948.604 $GPGSA,M,3,21,31,09,16,03,01,26,04,22,,,,1.7,0.8,1.5*39
+1674228948.634 $INHDT,179.38,T*11
+1674228948.644 $PSXN,20,0,0,0,0*3B
+1674228948.654 $PSXN,23,-0.27,0.41,179.38,0.01*10
+1674228948.674 $PSXN,26,0,44.5000,0.7800,-0.9000,NRP*6D
+1674228948.694 $PSXN,26,1,44.5000,0.7800,-0.9000,Monitoring Point #1*44
+1674228948.724 $PSXN,26,2,44.5000,0.7800,-0.9000,Monitoring Point #2*1F
+1674228948.754 $PSXN,26,3,44.5000,0.7800,-0.9000,Monitoring Point #3*33
+1674228948.784 $PSXN,26,4,44.5000,0.7800,-0.9000,Monitoring Point #4*68
+1674228948.814 $PSXN,26,5,44.5000,0.7800,-0.9000,Monitoring Point #5*3D
+1674228948.844 $PSXN,26,6,44.5000,0.7800,-0.9000,Monitoring Point #6*11
+1674228948.874 $PSXN,26,7,44.5000,0.7800,-0.9000,Monitoring Point #7*4A
+1674228948.904 $PSXN,26,8,44.5000,0.7800,-0.9000,Monitoring Point #8*64

shipgrav/tests/ex_files/TN400_dgs_proc.Raw

@@ -0,0 +1,2 @@
+03/12/2022,16:04:07.033,9984.869108166342,9995.951860660536,  0.031769279452,  0.049233442454, -0.894949104184, 62.394784400000,           16730,         4086876, 30.259833080000, 50.442681000000,  0.000020000000,  0.035710000000,  0.000070000000, -0.000010000000,  0.000000000000,  0.000000000000,  0.000000000000,  0.000000000000,  0.044215194148,2022,03,12,16,04,19.80,5370472
+03/12/2022,16:04:08.017,9984.870433770533,9977.883336291710,  0.030815605022,  0.037431721381, -0.894945527905, 62.394784400000,           16730,         4086005, 30.260701895000, 50.447645000000,  0.000020000000,  0.032780000000,  0.000020000000, -0.000010000000,  0.000000000000,  0.000000000000,  0.000000000000,  0.000000000000,  0.044061782566,2022,03,12,16,04,20.80,5370473

shipgrav/tests/test_grav_nodata.py

@@ -36,9 +36,26 @@ def test_plate_Z(self):
 
     def test_crustalthickness(self):
         rng = np.random.default_rng(123)  # seeded
-        C = sgg.crustal_thickness_2D(10*rng.random(1000))
+        signal = 10*rng.random(1000)
+        C = sgg.crustal_thickness_2D(signal)
         self.assertTrue(np.real(C[0])[0] - 0.04019 < 0.001)
 
+        C2 = sgg.crustal_thickness_2D(signal, back=True)
+        self.assertEqual(np.real(C[0])[0], np.real(C2[0][-1])[0])
+
+    def test_grav2d_folding(self):
+        rng = np.random.default_rng(123)  # seeded
+        X = np.arange(5); Y = np.arange(5)
+        Z = rng.random((5,5))
+        sdat = sgg.grav2d_folding(X, Y, Z, 100, 100,  drho=0.6, dz=6000, ifold=True, npower=5)
+
+        self.assertTrue(sdat[0,0] + 0.0043244755 < 0.001)
+
+    def test_grav2d_layer(self):
+        rng = np.random.default_rng(123)  # seeded
+        rho = 1e4*rng.random((5,5))
+        sdat = sgg.grav2d_layer_variable_density(rho, 100, 100, 3, 5)
+        self.assertTrue(sdat[0,0] - 130.877533 < 0.001)
 
 def suite():
     return unittest.makeSuite(gravNoDataTestCase, 'test')