Skip to content

Commit

Permalink
fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@TrevorBurgoyne
TrevorBurgoyne committed Oct 25, 2024
1 parent 1091165 commit d71bf7a
Show file tree
Hide file tree
Showing 2 changed files with 32 additions and 32 deletions.
18 changes: 9 additions & 9 deletions S3MP/mirror_path.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import psutil
from tqdm import tqdm

from s3mp.global_config import s3mpConfig
from s3mp.global_config import S3MPConfig
from s3mp.keys import KeySegment, get_matching_s3_keys
from s3mp.utils.local_file_utils import (
DEFAULT_LOAD_LEDGER,
Expand Down Expand Up @@ -39,7 +39,7 @@ def __init__(
self.key_segments: List[KeySegment] = [seg.__copy__() for seg in key_segments]
self._local_path_override: Path = None

self.mirror_root = mirror_root or s3mpConfig.mirror_root
self.mirror_root = mirror_root or S3MPConfig.mirror_root

@property
def s3_key(self) -> str:
Expand All @@ -51,7 +51,7 @@ def s3_key(self) -> str:
@property
def local_path(self) -> Path:
"""Get local path."""
return self._local_path_override or Path(s3mpConfig.mirror_root) / self.s3_key
return self._local_path_override or Path(S3MPConfig.mirror_root) / self.s3_key

def override_local_path(self, local_path: Path):
"""Override local path."""
Expand All @@ -69,7 +69,7 @@ def from_local_path(
local_path: Path, mirror_root: Path = None, **kwargs: Dict
) -> MirrorPath:
"""Create a MirrorPath from a local path."""
mirror_root = mirror_root or s3mpConfig.mirror_root
mirror_root = mirror_root or S3MPConfig.mirror_root
s3_key = local_path.relative_to(mirror_root).as_posix()
return MirrorPath.from_s3_key(s3_key, **kwargs)

Expand All @@ -95,8 +95,8 @@ def is_file_on_s3(self) -> bool:

def update_callback_on_skipped_transfer(self):
"""Update the current global callback if the transfer gets skipped."""
if s3mpConfig.callback and self in s3mpConfig.callback._transfer_objs:
s3mpConfig.callback(self.local_path.stat().st_size)
if S3MPConfig.callback and self in S3MPConfig.callback._transfer_objs:
S3MPConfig.callback(self.local_path.stat().st_size)

def download_to_mirror(self, overwrite: bool = False):
"""Download S3 file to mirror."""
Expand Down Expand Up @@ -236,9 +236,9 @@ def save_local(

def copy_to_mp_s3_only(self, dest_mp: MirrorPath):
"""Copy this file from S3 to a destination on S3."""
s3mpConfig.s3_client.copy_object(
CopySource={"Bucket": s3mpConfig.default_bucket_key, "Key": self.s3_key},
Bucket=s3mpConfig.default_bucket_key,
S3MPConfig.s3_client.copy_object(
CopySource={"Bucket": S3MPConfig.default_bucket_key, "Key": self.s3_key},
Bucket=S3MPConfig.default_bucket_key,
Key=dest_mp.s3_key,
)

Expand Down
46 changes: 23 additions & 23 deletions S3MP/utils/s3_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import warnings
from pathlib import Path

from s3mp.global_config import s3mpConfig
from s3mp.global_config import S3MPConfig
from s3mp.types import S3Bucket, S3Client, S3ListObjectV2Output


Expand All @@ -12,8 +12,8 @@ def s3_list_single_key(
client: S3Client = None,
) -> S3ListObjectV2Output:
"""List details of a single key on S3."""
bucket = bucket or s3mpConfig.bucket
client = client or s3mpConfig.s3_client
bucket = bucket or S3MPConfig.bucket
client = client or S3MPConfig.s3_client
return client.list_objects_v2(
Bucket=bucket.name, Prefix=key, Delimiter="/", MaxKeys=1
)
Expand All @@ -28,8 +28,8 @@ def s3_list_child_keys(
"""List details of all child keys on S3."""
if not key.endswith("/"):
warnings.warn(f"Listing child keys of {key} - key does not end with '/'")
bucket = bucket or s3mpConfig.bucket
client = client or s3mpConfig.s3_client
bucket = bucket or S3MPConfig.bucket
client = client or S3MPConfig.s3_client
params = {
"Bucket": bucket.name,
"Prefix": key,
Expand All @@ -47,16 +47,16 @@ def download_key(
client: S3Client = None,
) -> None:
"""Download a key from S3."""
bucket = bucket or s3mpConfig.bucket
client = client or s3mpConfig.s3_client
bucket = bucket or S3MPConfig.bucket
client = client or S3MPConfig.s3_client
if key_is_file_on_s3(key, bucket, client):
local_path.parent.mkdir(parents=True, exist_ok=True)
client.download_file(
bucket.name,
key,
str(local_path),
Callback=s3mpConfig.callback,
Config=s3mpConfig.transfer_config,
Callback=S3MPConfig.callback,
Config=S3MPConfig.transfer_config,
)
else:
for obj in s3_list_child_keys(key, bucket, client)["Contents"]:
Expand All @@ -70,15 +70,15 @@ def upload_to_key(
client: S3Client = None,
) -> None:
"""Upload a file or folder to a key on S3."""
bucket = bucket or s3mpConfig.bucket
client = client or s3mpConfig.s3_client
bucket = bucket or S3MPConfig.bucket
client = client or S3MPConfig.s3_client
if local_path.is_file():
client.upload_file(
str(local_path),
bucket.name,
key,
Callback=s3mpConfig.callback,
Config=s3mpConfig.transfer_config,
Callback=S3MPConfig.callback,
Config=S3MPConfig.transfer_config,
)
else:
for child in local_path.iterdir():
Expand All @@ -91,8 +91,8 @@ def key_exists_on_s3(
client: S3Client = None,
) -> bool:
"""Check if a key exists on S3."""
bucket = bucket or s3mpConfig.bucket
client = client or s3mpConfig.s3_client
bucket = bucket or S3MPConfig.bucket
client = client or S3MPConfig.s3_client
res = s3_list_single_key(key, bucket, client)
return "Contents" in res or "CommonPrefixes" in res

Expand All @@ -103,8 +103,8 @@ def key_is_file_on_s3(
client: S3Client = None,
) -> bool:
"""Check if a key is a file on S3, returns false if it is a folder. Raises an error if the key does not exist."""
bucket = bucket or s3mpConfig.bucket
client = client or s3mpConfig.s3_client
bucket = bucket or S3MPConfig.bucket
client = client or S3MPConfig.s3_client
if not key_exists_on_s3(key, bucket, client):
raise ValueError(f"Key {key} does not exist on S3")
res = s3_list_single_key(key, bucket, client)
Expand All @@ -124,8 +124,8 @@ def key_size_on_s3(
client: S3Client = None,
) -> int:
"""Get the size of a key on S3. Raises an error if the key does not exist."""
bucket = bucket or s3mpConfig.bucket
client = client or s3mpConfig.s3_client
bucket = bucket or S3MPConfig.bucket
client = client or S3MPConfig.s3_client
if not key_exists_on_s3(key, bucket, client):
raise ValueError(f"Key {key} does not exist on S3")
res = s3_list_single_key(key, bucket, client)
Expand All @@ -138,8 +138,8 @@ def delete_child_keys_on_s3(
client: S3Client = None,
) -> None:
"""Delete all keys that are children of a key on S3."""
bucket = bucket or s3mpConfig.bucket
client = client or s3mpConfig.s3_client
bucket = bucket or S3MPConfig.bucket
client = client or S3MPConfig.s3_client
for obj in s3_list_child_keys(key, bucket, client)["Contents"]:
client.delete_object(Bucket=bucket.name, Key=obj["Key"])

Expand All @@ -150,8 +150,8 @@ def delete_key_on_s3(
client: S3Client = None,
) -> None:
"""Delete a key on S3."""
bucket = bucket or s3mpConfig.bucket
client = client or s3mpConfig.s3_client
bucket = bucket or S3MPConfig.bucket
client = client or S3MPConfig.s3_client
if not key_exists_on_s3(key, bucket, client):
return
if key_is_file_on_s3(key, bucket, client):
Expand Down

0 comments on commit d71bf7a

Please sign in to comment.