From d71bf7a5f40fe6aeb9b507c4c3b36e0545f774c5 Mon Sep 17 00:00:00 2001 From: TrevorBurgoyne Date: Fri, 25 Oct 2024 10:02:37 -0500 Subject: [PATCH] fixes --- S3MP/mirror_path.py | 18 ++++++++--------- S3MP/utils/s3_utils.py | 46 +++++++++++++++++++++--------------------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/S3MP/mirror_path.py b/S3MP/mirror_path.py index f17221f..39cdcd5 100644 --- a/S3MP/mirror_path.py +++ b/S3MP/mirror_path.py @@ -9,7 +9,7 @@ import psutil from tqdm import tqdm -from s3mp.global_config import s3mpConfig +from s3mp.global_config import S3MPConfig from s3mp.keys import KeySegment, get_matching_s3_keys from s3mp.utils.local_file_utils import ( DEFAULT_LOAD_LEDGER, @@ -39,7 +39,7 @@ def __init__( self.key_segments: List[KeySegment] = [seg.__copy__() for seg in key_segments] self._local_path_override: Path = None - self.mirror_root = mirror_root or s3mpConfig.mirror_root + self.mirror_root = mirror_root or S3MPConfig.mirror_root @property def s3_key(self) -> str: @@ -51,7 +51,7 @@ def s3_key(self) -> str: @property def local_path(self) -> Path: """Get local path.""" - return self._local_path_override or Path(s3mpConfig.mirror_root) / self.s3_key + return self._local_path_override or Path(S3MPConfig.mirror_root) / self.s3_key def override_local_path(self, local_path: Path): """Override local path.""" @@ -69,7 +69,7 @@ def from_local_path( local_path: Path, mirror_root: Path = None, **kwargs: Dict ) -> MirrorPath: """Create a MirrorPath from a local path.""" - mirror_root = mirror_root or s3mpConfig.mirror_root + mirror_root = mirror_root or S3MPConfig.mirror_root s3_key = local_path.relative_to(mirror_root).as_posix() return MirrorPath.from_s3_key(s3_key, **kwargs) @@ -95,8 +95,8 @@ def is_file_on_s3(self) -> bool: def update_callback_on_skipped_transfer(self): """Update the current global callback if the transfer gets skipped.""" - if s3mpConfig.callback and self in s3mpConfig.callback._transfer_objs: - s3mpConfig.callback(self.local_path.stat().st_size) + if S3MPConfig.callback and self in S3MPConfig.callback._transfer_objs: + S3MPConfig.callback(self.local_path.stat().st_size) def download_to_mirror(self, overwrite: bool = False): """Download S3 file to mirror.""" @@ -236,9 +236,9 @@ def save_local( def copy_to_mp_s3_only(self, dest_mp: MirrorPath): """Copy this file from S3 to a destination on S3.""" - s3mpConfig.s3_client.copy_object( - CopySource={"Bucket": s3mpConfig.default_bucket_key, "Key": self.s3_key}, - Bucket=s3mpConfig.default_bucket_key, + S3MPConfig.s3_client.copy_object( + CopySource={"Bucket": S3MPConfig.default_bucket_key, "Key": self.s3_key}, + Bucket=S3MPConfig.default_bucket_key, Key=dest_mp.s3_key, ) diff --git a/S3MP/utils/s3_utils.py b/S3MP/utils/s3_utils.py index affc80f..87bea3a 100644 --- a/S3MP/utils/s3_utils.py +++ b/S3MP/utils/s3_utils.py @@ -2,7 +2,7 @@ import warnings from pathlib import Path -from s3mp.global_config import s3mpConfig +from s3mp.global_config import S3MPConfig from s3mp.types import S3Bucket, S3Client, S3ListObjectV2Output @@ -12,8 +12,8 @@ def s3_list_single_key( client: S3Client = None, ) -> S3ListObjectV2Output: """List details of a single key on S3.""" - bucket = bucket or s3mpConfig.bucket - client = client or s3mpConfig.s3_client + bucket = bucket or S3MPConfig.bucket + client = client or S3MPConfig.s3_client return client.list_objects_v2( Bucket=bucket.name, Prefix=key, Delimiter="/", MaxKeys=1 ) @@ -28,8 +28,8 @@ def s3_list_child_keys( """List details of all child keys on S3.""" if not key.endswith("/"): warnings.warn(f"Listing child keys of {key} - key does not end with '/'") - bucket = bucket or s3mpConfig.bucket - client = client or s3mpConfig.s3_client + bucket = bucket or S3MPConfig.bucket + client = client or S3MPConfig.s3_client params = { "Bucket": bucket.name, "Prefix": key, @@ -47,16 +47,16 @@ def download_key( client: S3Client = None, ) -> None: """Download a key from S3.""" - bucket = bucket or s3mpConfig.bucket - client = client or s3mpConfig.s3_client + bucket = bucket or S3MPConfig.bucket + client = client or S3MPConfig.s3_client if key_is_file_on_s3(key, bucket, client): local_path.parent.mkdir(parents=True, exist_ok=True) client.download_file( bucket.name, key, str(local_path), - Callback=s3mpConfig.callback, - Config=s3mpConfig.transfer_config, + Callback=S3MPConfig.callback, + Config=S3MPConfig.transfer_config, ) else: for obj in s3_list_child_keys(key, bucket, client)["Contents"]: @@ -70,15 +70,15 @@ def upload_to_key( client: S3Client = None, ) -> None: """Upload a file or folder to a key on S3.""" - bucket = bucket or s3mpConfig.bucket - client = client or s3mpConfig.s3_client + bucket = bucket or S3MPConfig.bucket + client = client or S3MPConfig.s3_client if local_path.is_file(): client.upload_file( str(local_path), bucket.name, key, - Callback=s3mpConfig.callback, - Config=s3mpConfig.transfer_config, + Callback=S3MPConfig.callback, + Config=S3MPConfig.transfer_config, ) else: for child in local_path.iterdir(): @@ -91,8 +91,8 @@ def key_exists_on_s3( client: S3Client = None, ) -> bool: """Check if a key exists on S3.""" - bucket = bucket or s3mpConfig.bucket - client = client or s3mpConfig.s3_client + bucket = bucket or S3MPConfig.bucket + client = client or S3MPConfig.s3_client res = s3_list_single_key(key, bucket, client) return "Contents" in res or "CommonPrefixes" in res @@ -103,8 +103,8 @@ def key_is_file_on_s3( client: S3Client = None, ) -> bool: """Check if a key is a file on S3, returns false if it is a folder. Raises an error if the key does not exist.""" - bucket = bucket or s3mpConfig.bucket - client = client or s3mpConfig.s3_client + bucket = bucket or S3MPConfig.bucket + client = client or S3MPConfig.s3_client if not key_exists_on_s3(key, bucket, client): raise ValueError(f"Key {key} does not exist on S3") res = s3_list_single_key(key, bucket, client) @@ -124,8 +124,8 @@ def key_size_on_s3( client: S3Client = None, ) -> int: """Get the size of a key on S3. Raises an error if the key does not exist.""" - bucket = bucket or s3mpConfig.bucket - client = client or s3mpConfig.s3_client + bucket = bucket or S3MPConfig.bucket + client = client or S3MPConfig.s3_client if not key_exists_on_s3(key, bucket, client): raise ValueError(f"Key {key} does not exist on S3") res = s3_list_single_key(key, bucket, client) @@ -138,8 +138,8 @@ def delete_child_keys_on_s3( client: S3Client = None, ) -> None: """Delete all keys that are children of a key on S3.""" - bucket = bucket or s3mpConfig.bucket - client = client or s3mpConfig.s3_client + bucket = bucket or S3MPConfig.bucket + client = client or S3MPConfig.s3_client for obj in s3_list_child_keys(key, bucket, client)["Contents"]: client.delete_object(Bucket=bucket.name, Key=obj["Key"]) @@ -150,8 +150,8 @@ def delete_key_on_s3( client: S3Client = None, ) -> None: """Delete a key on S3.""" - bucket = bucket or s3mpConfig.bucket - client = client or s3mpConfig.s3_client + bucket = bucket or S3MPConfig.bucket + client = client or S3MPConfig.s3_client if not key_exists_on_s3(key, bucket, client): return if key_is_file_on_s3(key, bucket, client):