Better resilience to missing start and end in find_filters() (#474)

storey/utils.py

@@ -217,7 +217,7 @@ def stringify_key(key_list):
 
 
 def _create_filter_tuple(dtime, attr, sign, list_tuples):
-    if attr:
+    if dtime is not None and attr:
         value = getattr(dtime, attr, None)
         tuple1 = (attr, sign, value)
         list_tuples.append(tuple1)
@@ -233,6 +233,8 @@ def _find_filter_helper(
     filter_column=None,
 ):
     single_filter = []
+    if dtime is None:
+        return
     if len(list_partitions) == 0 or first_uncommon is None:
         return
     last_partition = list_partitions[-1]
@@ -257,10 +259,12 @@ def _find_filter_helper(
 
 
 def _get_filters_for_filter_column(start, end, filter_column, side_range):
-    lower_limit_tuple = (filter_column, ">", start)
-    upper_limit_tuple = (filter_column, "<=", end)
-    side_range.append(lower_limit_tuple)
-    side_range.append(upper_limit_tuple)
+    if start is not None:
+        lower_limit_tuple = (filter_column, ">", start)
+        side_range.append(lower_limit_tuple)
+    if end is not None:
+        upper_limit_tuple = (filter_column, "<=", end)
+        side_range.append(upper_limit_tuple)
 
 
 def find_partitions(url, fs):

tests/test_utils.py

@@ -12,9 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import datetime
+
 from fsspec.implementations.local import LocalFileSystem
 
-from storey.utils import get_remaining_path, url_to_file_system
+from storey.utils import find_filters, get_remaining_path, url_to_file_system
 
 
 def test_get_path_utils():
@@ -29,3 +31,12 @@ def test_ds_get_path_utils():
     fs, path = url_to_file_system(url, "")
     assert path == "/path/to/object.csv"
     assert isinstance(fs, LocalFileSystem)
+
+
+def test_find_filters():
+    filters = []
+    find_filters([], datetime.datetime.min, datetime.datetime.max, filters, "time")
+    assert filters == [[("time", ">", datetime.datetime.min), ("time", "<=", datetime.datetime.max)]]
+    filters = []
+    find_filters([], None, datetime.datetime.max, filters, "time")
+    assert filters == [[("time", "<=", datetime.datetime.max)]]