From df32ca4255494a5a1e0af484efc692cfcf1332d0 Mon Sep 17 00:00:00 2001 From: Florent Vennetier Date: Thu, 13 Feb 2020 17:41:03 +0100 Subject: [PATCH] Save S3 bucket name in container metadata This will be used by oio-sds' account service when aggregating container statistics to make bucket statistics. --- .../common/middleware/autocontainerbase.py | 4 ++- oioswift/proxy/controllers/container.py | 9 +++++- oioswift/proxy/controllers/obj.py | 29 +++++++++++++------ oioswift/utils.py | 2 ++ tests/unit/controllers/test_obj.py | 2 +- 5 files changed, 34 insertions(+), 12 deletions(-) diff --git a/oioswift/common/middleware/autocontainerbase.py b/oioswift/common/middleware/autocontainerbase.py index 9e1f3994..45f2065d 100644 --- a/oioswift/common/middleware/autocontainerbase.py +++ b/oioswift/common/middleware/autocontainerbase.py @@ -85,7 +85,9 @@ def _extract_path(self, path): def _save_bucket_name(self, env): req = Request(env) - account, container, obj = self._extract_path(req.path_info) + # XXX(FVE): we should probably filter the name we extract here, + # it may contain MULTIUPLOAD_SUFFIX or whatever. + _account, container, _obj = self._extract_path(req.path_info) sys_meta_key = '%soio-bucket-name' % get_sys_meta_prefix('object') req.headers[sys_meta_key] = container diff --git a/oioswift/proxy/controllers/container.py b/oioswift/proxy/controllers/container.py index b6583937..d6f4eb6b 100644 --- a/oioswift/proxy/controllers/container.py +++ b/oioswift/proxy/controllers/container.py @@ -43,7 +43,7 @@ from oioswift.utils import \ handle_oio_no_such_container, handle_oio_timeout, \ - handle_service_busy, REQID_HEADER + handle_service_busy, REQID_HEADER, BUCKET_NAME_PROP, MULTIUPLOAD_SUFFIX class ContainerController(SwiftContainerController): @@ -283,6 +283,13 @@ def _convert_policy(self, req): def get_container_create_resp(self, req, headers): properties, system = self.properties_from_headers(headers) + # Save the name of the S3 bucket in a container property. + # This will be used when aggregating container statistics + # to make bucket statistics. + bname = self.container_name + if bname.endswith(MULTIUPLOAD_SUFFIX): + bname = bname[:-len(MULTIUPLOAD_SUFFIX)] + system[BUCKET_NAME_PROP] = bname # TODO container update metadata oio_headers = {REQID_HEADER: self.trans_id} created = self.app.storage.container_create( diff --git a/oioswift/proxy/controllers/obj.py b/oioswift/proxy/controllers/obj.py index c3c4a742..1d85d2b9 100644 --- a/oioswift/proxy/controllers/obj.py +++ b/oioswift/proxy/controllers/obj.py @@ -56,9 +56,10 @@ from oio.common.exceptions import SourceReadTimeout from oioswift.utils import check_if_none_match, \ handle_not_allowed, handle_oio_timeout, handle_service_busy, \ - REQID_HEADER + REQID_HEADER, BUCKET_NAME_PROP, MULTIUPLOAD_SUFFIX SLO = 'x-static-large-object' +BUCKET_NAME_HEADER = 'X-Object-Sysmeta-Oio-Bucket-Name' class ObjectControllerRouter(object): @@ -202,7 +203,7 @@ def enforce_versioning(self, req): if not SUPPORT_VERSIONING: return None - root_container = req.headers.get('X-Object-Sysmeta-Oio-Bucket-Name') + root_container = req.headers.get(BUCKET_NAME_HEADER) if root_container is None: return None @@ -251,8 +252,8 @@ def get_object_head_resp(self, req): headers=oio_headers, force_master=force_master) break except (exceptions.NoSuchObject, exceptions.NoSuchContainer): - if force_master \ - or not self.container_name.endswith('+segments'): + if force_master or not \ + self.container_name.endswith(MULTIUPLOAD_SUFFIX): # Either the request failed with the master, # or it is not an MPU return HTTPNotFound(request=req) @@ -303,8 +304,8 @@ def get_object_fetch_resp(self, req): force_master=force_master) break except (exceptions.NoSuchObject, exceptions.NoSuchContainer): - if force_master \ - or not self.container_name.endswith('+segments'): + if force_master or not \ + self.container_name.endswith(MULTIUPLOAD_SUFFIX): # Either the request failed with the master, # or it is not an MPU return HTTPNotFound(request=req) @@ -622,7 +623,6 @@ def _object_create(self, account, container, **kwargs): def _store_object(self, req, data_source, headers): content_type = req.headers.get('content-type', 'octet/stream') - storage = self.app.storage policy = None container_info = self.container_info(self.account_name, self.container_name, req) @@ -645,24 +645,35 @@ def _store_object(self, req, data_source, headers): content_length = int(req.headers.get('content-length', 0)) policy = self._get_auto_policy_from_size(content_length) + ct_props = {'properties': {}, 'system': {}} metadata = self.load_object_metadata(headers) oio_headers = {REQID_HEADER: self.trans_id} # only send headers if needed if SUPPORT_VERSIONING and headers.get(FORCEVERSIONING_HEADER): oio_headers[FORCEVERSIONING_HEADER] = \ headers.get(FORCEVERSIONING_HEADER) + # In case a shard is being created, save the name of the S3 bucket + # in a container property. This will be used when aggregating + # container statistics to make bucket statistics. + if BUCKET_NAME_HEADER in headers: + bname = headers[BUCKET_NAME_HEADER] + # FIXME(FVE): the segments container is not part of another bucket! + # We should not have to strip this here. + if bname and bname.endswith(MULTIUPLOAD_SUFFIX): + bname = bname[:-len(MULTIUPLOAD_SUFFIX)] + ct_props['system'][BUCKET_NAME_PROP] = bname try: _chunks, _size, checksum, _meta = self._object_create( self.account_name, self.container_name, obj_name=self.object_name, file_or_path=data_source, mime_type=content_type, policy=policy, headers=oio_headers, etag=req.headers.get('etag', '').strip('"'), - properties=metadata) + properties=metadata, container_properties=ct_props) # TODO(FVE): when oio-sds supports it, do that in a callback # passed to object_create (or whatever upload method supports it) footer_md = self.load_object_metadata(self._get_footers(req)) if footer_md: - storage.object_set_properties( + self.app.storage.object_set_properties( self.account_name, self.container_name, self.object_name, version=_meta.get('version', None), properties=footer_md) except exceptions.Conflict: diff --git a/oioswift/utils.py b/oioswift/utils.py index 28cac365..faf77433 100644 --- a/oioswift/utils.py +++ b/oioswift/utils.py @@ -34,6 +34,8 @@ REQID_HEADER = 'x-oio-req-id' +BUCKET_NAME_PROP = "sys.m2.bucket.name" +MULTIUPLOAD_SUFFIX = '+segments' _FORMAT_MAP = {"xml": 'application/xml', "json": 'application/json', "plain": 'text/plain'} diff --git a/tests/unit/controllers/test_obj.py b/tests/unit/controllers/test_obj.py index c6120e0d..eec1351d 100644 --- a/tests/unit/controllers/test_obj.py +++ b/tests/unit/controllers/test_obj.py @@ -203,7 +203,7 @@ def test_PUT_simple(self): 'a', 'c', obj_name='o', etag='', properties={}, mime_type='application/octet-stream', file_or_path=req.environ['wsgi.input'], policy=None, - headers=ANY) + headers=ANY, container_properties=ANY) self.assertEqual(201, resp.status_int) self.assertIn('Last-Modified', resp.headers) self.assertIn('Etag', resp.headers)