Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into fix/bucket-name-restrictions
Browse files Browse the repository at this point in the history
ppolewicz authored May 29, 2021
2 parents ccbc0d8 + cf16a08 commit 9563c96
Showing 43 changed files with 1,354 additions and 647 deletions.
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -6,6 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Added
* `ScanPoliciesManager` is able to filter b2 files by upload timestamp

### Changed
* `Synchronizer.make_file_sync_actions` and `Synchronizer.make_folder_sync_actions` were made private in v2 interface
* Refactored `sync.file.*File` and `sync.file.*FileVersion` to `sync.path.*SyncPath`
* Refactored `FileVersionInfo` to `FileVersion`
* `ScanPoliciesManager` exclusion interface changed

### Fixed
* Fix call to incorrect internal api in `B2Api.get_download_url_for_file_name`

7 changes: 3 additions & 4 deletions b2sdk/_v2/__init__.py
Original file line number Diff line number Diff line change
@@ -63,8 +63,8 @@
# data classes

from b2sdk.file_version import FileIdAndName
from b2sdk.file_version import FileVersionInfo
from b2sdk.file_version import FileVersionInfoFactory
from b2sdk.file_version import FileVersion
from b2sdk.file_version import FileVersionFactory
from b2sdk.large_file.part import Part
from b2sdk.large_file.unfinished_large_file import UnfinishedLargeFile

@@ -160,12 +160,11 @@
from b2sdk.sync.exception import EnvironmentEncodingError
from b2sdk.sync.exception import IncompleteSync
from b2sdk.sync.exception import InvalidArgument
from b2sdk.sync.file import File, B2File
from b2sdk.sync.file import FileVersion, B2FileVersion
from b2sdk.sync.folder import AbstractFolder
from b2sdk.sync.folder import B2Folder
from b2sdk.sync.folder import LocalFolder
from b2sdk.sync.folder_parser import parse_sync_folder
from b2sdk.sync.path import AbstractSyncPath, B2SyncPath, LocalSyncPath
from b2sdk.sync.policy import AbstractFileSyncPolicy
from b2sdk.sync.policy import CompareVersionMode
from b2sdk.sync.policy import NewerFileSyncMode
34 changes: 17 additions & 17 deletions b2sdk/bucket.py
Original file line number Diff line number Diff line change
@@ -21,7 +21,7 @@
UNKNOWN_BUCKET_RETENTION,
LegalHold,
)
from .file_version import FileVersionInfo, FileVersionInfoFactory
from .file_version import FileVersion, FileVersionFactory
from .progress import DoNothingProgressListener
from .transfer.emerge.executor import AUTO_CONTENT_TYPE
from .transfer.emerge.write_intent import WriteIntent
@@ -39,7 +39,7 @@ class Bucket(metaclass=B2TraceMeta):
"""

DEFAULT_CONTENT_TYPE = AUTO_CONTENT_TYPE
FILE_VERSION_FACTORY = staticmethod(FileVersionInfoFactory)
FILE_VERSION_FACTORY = staticmethod(FileVersionFactory)

def __init__(
self,
@@ -224,18 +224,18 @@ def download_file_by_name(
encryption=encryption,
)

def get_file_info_by_id(self, file_id: str) -> FileVersionInfo:
def get_file_info_by_id(self, file_id: str) -> FileVersion:
"""
Gets a file version's info by ID.
Gets a file version's by ID.
:param str file_id: the id of the file who's info will be retrieved.
:rtype: generator[b2sdk.v1.FileVersionInfo]
"""
return self.FILE_VERSION_FACTORY.from_api_response(self.api.get_file_info(file_id))

def get_file_info_by_name(self, file_name: str) -> FileVersionInfo:
def get_file_info_by_name(self, file_name: str) -> FileVersion:
"""
Gets a file version's info by its name.
Gets a file version's by its name.
:param str file_name: the name of the file who's info will be retrieved.
:rtype: generator[b2sdk.v1.FileVersionInfo]
@@ -290,11 +290,11 @@ def list_file_versions(self, file_name, fetch_count=None):
)

for entry in response['files']:
file_version_info = self.FILE_VERSION_FACTORY.from_api_response(entry)
if file_version_info.file_name != file_name:
file_version = self.FILE_VERSION_FACTORY.from_api_response(entry)
if file_version.file_name != file_name:
# All versions for the requested file name have been listed.
return
yield file_version_info
yield file_version
start_file_name = response['nextFileName']
start_file_id = response['nextFileId']
if start_file_name is None:
@@ -319,7 +319,7 @@ def ls(self, folder_to_list='', show_versions=False, recursive=False, fetch_coun
:param bool recursive: if ``True``, list folders recursively
:param int,None fetch_count: how many entries to return or ``None`` to use the default. Acceptable values: 1 - 10000
:rtype: generator[tuple[b2sdk.v1.FileVersionInfo, str]]
:returns: generator of (file_version_info, folder_name) tuples
:returns: generator of (file_version, folder_name) tuples
.. note::
In case of `recursive=True`, folder_name is returned only for first file in the folder.
@@ -350,15 +350,15 @@ def ls(self, folder_to_list='', show_versions=False, recursive=False, fetch_coun
else:
response = session.list_file_names(self.id_, start_file_name, fetch_count, prefix)
for entry in response['files']:
file_version_info = self.FILE_VERSION_FACTORY.from_api_response(entry)
if not file_version_info.file_name.startswith(prefix):
file_version = self.FILE_VERSION_FACTORY.from_api_response(entry)
if not file_version.file_name.startswith(prefix):
# We're past the files we care about
return
after_prefix = file_version_info.file_name[len(prefix):]
after_prefix = file_version.file_name[len(prefix):]
if '/' not in after_prefix or recursive:
# This is not a folder, so we'll print it out and
# continue on.
yield file_version_info, None
yield file_version, None
current_dir = None
else:
# This is a folder. If it's different than the folder
@@ -368,7 +368,7 @@ def ls(self, folder_to_list='', show_versions=False, recursive=False, fetch_coun
folder_with_slash = after_prefix.split('/')[0] + '/'
if folder_with_slash != current_dir:
folder_name = prefix + folder_with_slash
yield file_version_info, folder_name
yield file_version, folder_name
current_dir = folder_with_slash
if response['nextFileName'] is None:
# The response says there are no more files in the bucket,
@@ -948,12 +948,12 @@ def from_api_bucket_dict(cls, api, bucket_dict):
}
},
"fileLockConfiguration": {
"isClientAuthorizedToRead": true,
"isClientAuthorizedToRead": true,
"value": {
"defaultRetention": {
"mode": null,
"period": null
},
},
"isFileLockEnabled": false
}
}
2 changes: 1 addition & 1 deletion b2sdk/encryption/setting.py
Original file line number Diff line number Diff line change
@@ -206,7 +206,7 @@ class EncryptionSettingFactory:
# if not authorized to read:
# isClientAuthorizedToRead is False and there is no value, so no mode
#
# BUT file_version_info (get_file_info, list_file_versions, upload_file etc)
# BUT file_version (get_file_info, list_file_versions, upload_file etc)
# if the file is encrypted, then
# "serverSideEncryption": {"algorithm": "AES256", "mode": "SSE-B2"},
# or
14 changes: 7 additions & 7 deletions b2sdk/exception.py
Original file line number Diff line number Diff line change
@@ -199,21 +199,21 @@ def should_retry_http(self):


class DestFileNewer(B2Error):
def __init__(self, dest_file, source_file, dest_prefix, source_prefix):
def __init__(self, dest_path, source_path, dest_prefix, source_prefix):
super(DestFileNewer, self).__init__()
self.dest_file = dest_file
self.source_file = source_file
self.dest_path = dest_path
self.source_path = source_path
self.dest_prefix = dest_prefix
self.source_prefix = source_prefix

def __str__(self):
return 'source file is older than destination: %s%s with a time of %s cannot be synced to %s%s with a time of %s, unless a valid newer_file_mode is provided' % (
self.source_prefix,
self.source_file.name,
self.source_file.latest_version().mod_time,
self.source_path.relative_path,
self.source_path.mod_time,
self.dest_prefix,
self.dest_file.name,
self.dest_file.latest_version().mod_time,
self.dest_path.relative_path,
self.dest_path.mod_time,
)

def should_retry_http(self):
51 changes: 29 additions & 22 deletions b2sdk/file_version.py
Original file line number Diff line number Diff line change
@@ -12,9 +12,10 @@

from .encryption.setting import EncryptionSetting, EncryptionSettingFactory
from .file_lock import FileRetentionSetting, LegalHold
from .raw_api import SRC_LAST_MODIFIED_MILLIS


class FileVersionInfo(object):
class FileVersion:
"""
A structure which represents a version of a file (in B2 cloud).
@@ -46,6 +47,7 @@ class FileVersionInfo(object):
'server_side_encryption',
'legal_hold',
'file_retention',
'mod_time_millis',
]

def __init__(
@@ -79,6 +81,11 @@ def __init__(
self.legal_hold = legal_hold
self.file_retention = file_retention

if SRC_LAST_MODIFIED_MILLIS in self.file_info:
self.mod_time_millis = int(self.file_info[SRC_LAST_MODIFIED_MILLIS])
else:
self.mod_time_millis = self.upload_timestamp

def as_dict(self):
""" represents the object as a dict which looks almost exactly like the raw api output for upload/list """
result = {
@@ -114,13 +121,13 @@ def __eq__(self, other):
return True


class FileVersionInfoFactory(object):
class FileVersionFactory(object):
"""
Construct :py:class:`b2sdk.v1.FileVersionInfo` objects from various structures.
"""

@classmethod
def from_api_response(cls, file_info_dict, force_action=None):
def from_api_response(cls, file_version_dict, force_action=None):
"""
Turn this:
@@ -153,28 +160,28 @@ def from_api_response(cls, file_info_dict, force_action=None):
into a :py:class:`b2sdk.v1.FileVersionInfo` object.
"""
assert file_info_dict.get('action') is None or force_action is None, \
assert file_version_dict.get('action') is None or force_action is None, \
'action was provided by both info_dict and function argument'
action = file_info_dict.get('action') or force_action
file_name = file_info_dict['fileName']
id_ = file_info_dict['fileId']
if 'size' in file_info_dict:
size = file_info_dict['size']
elif 'contentLength' in file_info_dict:
size = file_info_dict['contentLength']
action = file_version_dict.get('action') or force_action
file_name = file_version_dict['fileName']
id_ = file_version_dict['fileId']
if 'size' in file_version_dict:
size = file_version_dict['size']
elif 'contentLength' in file_version_dict:
size = file_version_dict['contentLength']
else:
raise ValueError('no size or contentLength')
upload_timestamp = file_info_dict.get('uploadTimestamp')
content_type = file_info_dict.get('contentType')
content_sha1 = file_info_dict.get('contentSha1')
content_md5 = file_info_dict.get('contentMd5')
file_info = file_info_dict.get('fileInfo')
server_side_encryption = EncryptionSettingFactory.from_file_version_dict(file_info_dict)
file_retention = FileRetentionSetting.from_file_version_dict(file_info_dict)
upload_timestamp = file_version_dict.get('uploadTimestamp')
content_type = file_version_dict.get('contentType')
content_sha1 = file_version_dict.get('contentSha1')
content_md5 = file_version_dict.get('contentMd5')
file_info = file_version_dict.get('fileInfo')
server_side_encryption = EncryptionSettingFactory.from_file_version_dict(file_version_dict)
file_retention = FileRetentionSetting.from_file_version_dict(file_version_dict)

legal_hold = LegalHold.from_file_version_dict(file_info_dict)
legal_hold = LegalHold.from_file_version_dict(file_version_dict)

return FileVersionInfo(
return FileVersion(
id_,
file_name,
size,
@@ -191,7 +198,7 @@ def from_api_response(cls, file_info_dict, force_action=None):

@classmethod
def from_cancel_large_file_response(cls, response):
return FileVersionInfo(
return FileVersion(
response['fileId'],
response['fileName'],
0, # size
@@ -204,7 +211,7 @@ def from_cancel_large_file_response(cls, response):

@classmethod
def from_response_headers(cls, headers):
return FileVersionInfo(
return FileVersion(
id_=headers.get('x-bz-file-id'),
file_name=headers.get('x-bz-file-name'),
size=headers.get('content-length'),
4 changes: 2 additions & 2 deletions b2sdk/large_file/services.py
Original file line number Diff line number Diff line change
@@ -12,7 +12,7 @@

from b2sdk.encryption.setting import EncryptionSetting
from b2sdk.file_lock import FileRetentionSetting, LegalHold
from b2sdk.file_version import FileVersionInfoFactory
from b2sdk.file_version import FileVersionFactory
from b2sdk.large_file.part import PartFactory
from b2sdk.large_file.unfinished_large_file import UnfinishedLargeFile

@@ -119,4 +119,4 @@ def cancel_large_file(self, file_id):
:rtype: None
"""
response = self.services.session.cancel_large_file(file_id)
return FileVersionInfoFactory.from_cancel_large_file_response(response)
return FileVersionFactory.from_cancel_large_file_response(response)
Loading

0 comments on commit 9563c96

Please sign in to comment.