Skip to content

Commit

Permalink
refactoring
Browse files Browse the repository at this point in the history
  • Loading branch information
ilongin committed Aug 16, 2024
1 parent 7d0f7f4 commit 2e94125
Showing 1 changed file with 4 additions and 5 deletions.
9 changes: 4 additions & 5 deletions src/datachain/lib/dc.py
Original file line number Diff line number Diff line change
Expand Up @@ -321,10 +321,10 @@ def _listing_filter(
def _file_c(name: str) -> C:
return C(f"{object_name}.{name}")

dc = dc.filter(C(f"{object_name}.is_latest") == true())
dc = dc.filter(_file_c("is_latest") == true())
if recursive:
root = False
where = C(f"{object_name}.path").glob(path)
where = _file_c("path").glob(path)
if not path or path == "/":
# root of the bucket, e.g s3://bucket/ -> getting all the nodes
# in the bucket
Expand All @@ -337,15 +337,14 @@ def _file_c(name: str) -> C:
# and we are adding a proper glob syntax for it
# e.g s3://bucket/dir1 -> s3://bucket/dir1/*
dir_path = path.rstrip("/") + "/*"
where = where | C(f"{object_name}.path").glob(dir_path)
where = where | _file_c("path").glob(dir_path)

if not root:
# not a root, so running glob query
dc = dc.filter(where)
else:
dc = dc.filter(
pathfunc.parent(C(f"{object_name}.path"))
== path.lstrip("/").rstrip("/*")
pathfunc.parent(_file_c("path")) == path.lstrip("/").rstrip("/*")
)

return dc
Expand Down

0 comments on commit 2e94125

Please sign in to comment.