From 58c7a2e1249168d064176b5e1ff803a3372a0749 Mon Sep 17 00:00:00 2001 From: Anton Sidelnikov <53078276+anton-sidelnikov@users.noreply.github.com> Date: Tue, 25 Feb 2025 13:40:18 +0100 Subject: [PATCH] [Fix] Obs multipart name extraction fix (#541) [Fix] Obs multipart name extraction fix Fix #538 and ##518 Reviewed-by: Artem Lifshits --- doc/source/sdk/guides/index.rst | 1 + doc/source/sdk/guides/obs.rst | 22 +++++++++ examples/obs/multipart_upload.py | 33 +++++++++++++ otcextensions/sdk/obs/v1/_proxy.py | 3 +- otcextensions/sdk/obs/v1/container.py | 48 ++++++++++++++++++- .../obs-multipart-fix-bc1cbba753bfafbd.yaml | 5 ++ 6 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 doc/source/sdk/guides/obs.rst create mode 100644 examples/obs/multipart_upload.py create mode 100644 releasenotes/notes/obs-multipart-fix-bc1cbba753bfafbd.yaml diff --git a/doc/source/sdk/guides/index.rst b/doc/source/sdk/guides/index.rst index fc0be7f76..d015b5851 100644 --- a/doc/source/sdk/guides/index.rst +++ b/doc/source/sdk/guides/index.rst @@ -27,6 +27,7 @@ Open Telekom Cloud related User Guides mrs modelarts nat + obs rds sfsturbo smn diff --git a/doc/source/sdk/guides/obs.rst b/doc/source/sdk/guides/obs.rst new file mode 100644 index 000000000..106b6bfbd --- /dev/null +++ b/doc/source/sdk/guides/obs.rst @@ -0,0 +1,22 @@ +Object Storage Service (OBS) +============================ + +Object Storage Service (OBS) is an object-based storage service +that provides customers with massive, secure, reliable, +and cost-effective data storage capabilities, such as bucket creation, +modification, and deletion, as well as object upload, download, and deletion. + +.. contents:: Table of Contents + :local: + +Object +------ + +Multipart upload +^^^^^^^^^^^^^^^^ + +This interface is used to create an OBS multipart upload. +:class:`~otcextensions.sdk.obs.v1.obj.Object`. + +.. literalinclude:: ../examples/obs/multipart_upload.py + :lines: 16-33 diff --git a/examples/obs/multipart_upload.py b/examples/obs/multipart_upload.py new file mode 100644 index 000000000..279a87a60 --- /dev/null +++ b/examples/obs/multipart_upload.py @@ -0,0 +1,33 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import openstack +from otcextensions import sdk +""" +Create an multipart upload of a large object +""" + +openstack.enable_logging(True) +conn = openstack.connect(cloud='otc') +sdk.register_otc_extensions(conn) +sdk.get_ak_sk(conn) + +container = conn.obs.create_container( + name='test-multipart', + storage_acl='public-read-write', + storage_class='STANDARD' +) + +obj = conn.obs.upload_object( + name='myfile.zip', + container=container, + segment_size=1400000 +) diff --git a/otcextensions/sdk/obs/v1/_proxy.py b/otcextensions/sdk/obs/v1/_proxy.py index c989b1d68..8d939a833 100644 --- a/otcextensions/sdk/obs/v1/_proxy.py +++ b/otcextensions/sdk/obs/v1/_proxy.py @@ -451,10 +451,11 @@ def _upload_large_object(self, endpoint, filename, headers, url = f'{endpoint}/{object_name}' # Schedule the segments for upload for name, segment in segments.items(): + part_number = name.rsplit('/', 1)[-1] # Async call to put - schedules execution and returns a future segment_future = self._connection._pool_executor.submit( self.put, - f'{url}?partNumber={name[-1]}&uploadId={upload_id}', + f'{url}?partNumber={part_number}&uploadId={upload_id}', headers=headers, data=segment, requests_auth=requests_auth, raise_exc=False) diff --git a/otcextensions/sdk/obs/v1/container.py b/otcextensions/sdk/obs/v1/container.py index 8af46301e..fe09ccae4 100644 --- a/otcextensions/sdk/obs/v1/container.py +++ b/otcextensions/sdk/obs/v1/container.py @@ -44,8 +44,46 @@ class Container(_base.BaseResource): name = resource.Body('Name', alternate_id=True, alias='id') creation_date = resource.Body('CreationDate') + # When creating a bucket, you can use this parameter + # to set a pre-defined ACL. storage_acl = resource.Header('x-amz-acl') - storage_class = resource.Header('x-default-storage-class') + # When creating a bucket, you can add this header + # to set the default storage class for the bucket. + # Value range: + # STANDARD (Standard storage) + # WARM (Warm storage) + # COLD (Cold storage) + storage_class = resource.Header('x-obs-storage-class') + # Grants the read permission to all users in a specified domain. + grant_read = resource.Header('x-obs-grant-read') + # Grants the WRITE permission to all users in a specified domain to create, + # delete, and overwrite all objects in a bucket; and initiate multipart + # uploads, upload parts, copy parts, assemble parts, + # and cancel multipart uploads. + grant_write = resource.Header('x-obs-grant-write') + # Grant the READ_ACP permission to all users in a specified domain + # to allow them to read the bucket ACL. + grant_read_acp = resource.Header('x-obs-grant-read-acp') + # Grants the WRITE_ACP permission to all users in a specified domain + # to allow them to modify the bucket ACL. + grant_write_acp = resource.Header('x-obs-grant-write-acp') + # Grants the FULL_CONTROL permission to all users in a specified domain. + grant_full_control = resource.Header('x-obs-grant-full-control') + # Grants the READ permission to all users in a specified domain. + # By default, the read permission is granted on all objects in the bucket. + grant_read_delivered = resource.Header('x-obs-grant-read-delivered') + # Grants the FULL_CONTROL permission to all users in a specified domain. + # By default, the FULL_CONTROL permission is granted on all + # objects in the bucket. + grant_full_delivered = resource.Header( + 'x-obs-grant-full-control-delivered' + ) + # This header can be carried when you want to create + # a parallel file system. + fs_file_interface = resource.Header('x-obs-fs-file-interface') + # When creating a bucket, + # you can use this header to enable WORM for the bucket. + object_lock_enabled = resource.Header('x-obs-bucket-object-lock-enabled') def _translate_response(self, response, has_body=True, error_message=None): """Given a KSA response, inflate this instance with its data @@ -87,6 +125,11 @@ def _prepare_request(self, requires_id=None, prepend_key=False): base_path = '/' headers = {} + for k, v in self._header.dirty.items(): + if isinstance(v, list): + headers[k] = ", ".join(v) + else: + headers[k] = str(v) uri = base_path % self._uri.attributes if requires_id: if self.id is None: @@ -135,7 +178,8 @@ def list(cls, session, paginated=False, return def create(self, session, prepend_key=True, - endpoint_override=None, headers=None, requests_auth=None): + endpoint_override=None, headers=None, requests_auth=None, + **attrs): if not self.allow_create: raise exceptions.MethodNotSupported(self, "create") diff --git a/releasenotes/notes/obs-multipart-fix-bc1cbba753bfafbd.yaml b/releasenotes/notes/obs-multipart-fix-bc1cbba753bfafbd.yaml new file mode 100644 index 000000000..73548da6b --- /dev/null +++ b/releasenotes/notes/obs-multipart-fix-bc1cbba753bfafbd.yaml @@ -0,0 +1,5 @@ +fixes: + - | + SDK OBS container headers assigning + - | + SDK OBS object multipart upload fix