Save S3 bucket name in container metadata

This will be used by oio-sds' account service when aggregating container
statistics to make bucket statistics.

oioswift/common/middleware/autocontainerbase.py

@@ -85,7 +85,9 @@ def _extract_path(self, path):
 
     def _save_bucket_name(self, env):
         req = Request(env)
-        account, container, obj = self._extract_path(req.path_info)
+        # XXX(FVE): we should probably filter the name we extract here,
+        # it may contain MULTIUPLOAD_SUFFIX or whatever.
+        _account, container, _obj = self._extract_path(req.path_info)
         sys_meta_key = '%soio-bucket-name' % get_sys_meta_prefix('object')
         req.headers[sys_meta_key] = container
 

oioswift/proxy/controllers/container.py

@@ -43,7 +43,7 @@
 
 from oioswift.utils import \
     handle_oio_no_such_container, handle_oio_timeout, \
-    handle_service_busy, REQID_HEADER
+    handle_service_busy, REQID_HEADER, BUCKET_NAME_PROP, MULTIUPLOAD_SUFFIX
 
 
 class ContainerController(SwiftContainerController):
@@ -283,6 +283,13 @@ def _convert_policy(self, req):
 
     def get_container_create_resp(self, req, headers):
         properties, system = self.properties_from_headers(headers)
+        # Save the name of the S3 bucket in a container property.
+        # This will be used when aggregating container statistics
+        # to make bucket statistics.
+        bname = self.container_name
+        if bname.endswith(MULTIUPLOAD_SUFFIX):
+            bname = bname[:-len(MULTIUPLOAD_SUFFIX)]
+        system[BUCKET_NAME_PROP] = bname
         # TODO container update metadata
         oio_headers = {REQID_HEADER: self.trans_id}
         created = self.app.storage.container_create(

oioswift/proxy/controllers/obj.py

@@ -56,9 +56,10 @@
 from oio.common.exceptions import SourceReadTimeout
 from oioswift.utils import check_if_none_match, \
     handle_not_allowed, handle_oio_timeout, handle_service_busy, \
-    REQID_HEADER
+    REQID_HEADER, BUCKET_NAME_PROP, MULTIUPLOAD_SUFFIX
 
 SLO = 'x-static-large-object'
+BUCKET_NAME_HEADER = 'X-Object-Sysmeta-Oio-Bucket-Name'
 
 
 class ObjectControllerRouter(object):
@@ -202,7 +203,7 @@ def enforce_versioning(self, req):
         if not SUPPORT_VERSIONING:
             return None
 
-        root_container = req.headers.get('X-Object-Sysmeta-Oio-Bucket-Name')
+        root_container = req.headers.get(BUCKET_NAME_HEADER)
         if root_container is None:
             return None
 
@@ -251,8 +252,8 @@ def get_object_head_resp(self, req):
                         headers=oio_headers, force_master=force_master)
                 break
             except (exceptions.NoSuchObject, exceptions.NoSuchContainer):
-                if force_master \
-                        or not self.container_name.endswith('+segments'):
+                if force_master or not \
+                        self.container_name.endswith(MULTIUPLOAD_SUFFIX):
                     # Either the request failed with the master,
                     # or it is not an MPU
                     return HTTPNotFound(request=req)
@@ -303,8 +304,8 @@ def get_object_fetch_resp(self, req):
                     force_master=force_master)
                 break
             except (exceptions.NoSuchObject, exceptions.NoSuchContainer):
-                if force_master \
-                        or not self.container_name.endswith('+segments'):
+                if force_master or not \
+                        self.container_name.endswith(MULTIUPLOAD_SUFFIX):
                     # Either the request failed with the master,
                     # or it is not an MPU
                     return HTTPNotFound(request=req)
@@ -622,7 +623,6 @@ def _object_create(self, account, container, **kwargs):
 
     def _store_object(self, req, data_source, headers):
         content_type = req.headers.get('content-type', 'octet/stream')
-        storage = self.app.storage
         policy = None
         container_info = self.container_info(self.account_name,
                                              self.container_name, req)
@@ -645,24 +645,35 @@ def _store_object(self, req, data_source, headers):
                 content_length = int(req.headers.get('content-length', 0))
                 policy = self._get_auto_policy_from_size(content_length)
 
+        ct_props = {'properties': {}, 'system': {}}
         metadata = self.load_object_metadata(headers)
         oio_headers = {REQID_HEADER: self.trans_id}
         # only send headers if needed
         if SUPPORT_VERSIONING and headers.get(FORCEVERSIONING_HEADER):
             oio_headers[FORCEVERSIONING_HEADER] = \
                 headers.get(FORCEVERSIONING_HEADER)
+        # In case a shard is being created, save the name of the S3 bucket
+        # in a container property. This will be used when aggregating
+        # container statistics to make bucket statistics.
+        if BUCKET_NAME_HEADER in headers:
+            bname = headers[BUCKET_NAME_HEADER]
+            # FIXME(FVE): the segments container is not part of another bucket!
+            # We should not have to strip this here.
+            if bname and bname.endswith(MULTIUPLOAD_SUFFIX):
+                bname = bname[:-len(MULTIUPLOAD_SUFFIX)]
+            ct_props['system'][BUCKET_NAME_PROP] = bname
         try:
             _chunks, _size, checksum, _meta = self._object_create(
                 self.account_name, self.container_name,
                 obj_name=self.object_name, file_or_path=data_source,
                 mime_type=content_type, policy=policy, headers=oio_headers,
                 etag=req.headers.get('etag', '').strip('"'),
-                properties=metadata)
+                properties=metadata, container_properties=ct_props)
             # TODO(FVE): when oio-sds supports it, do that in a callback
             # passed to object_create (or whatever upload method supports it)
             footer_md = self.load_object_metadata(self._get_footers(req))
             if footer_md:
-                storage.object_set_properties(
+                self.app.storage.object_set_properties(
                     self.account_name, self.container_name, self.object_name,
                     version=_meta.get('version', None), properties=footer_md)
         except exceptions.Conflict:

oioswift/utils.py

@@ -34,6 +34,8 @@
     REQID_HEADER = 'x-oio-req-id'
 
 
+BUCKET_NAME_PROP = "sys.m2.bucket.name"
+MULTIUPLOAD_SUFFIX = '+segments'
 _FORMAT_MAP = {"xml": 'application/xml', "json": 'application/json',
                "plain": 'text/plain'}
 

tests/unit/controllers/test_obj.py

@@ -203,7 +203,7 @@ def test_PUT_simple(self):
                      'a', 'c', obj_name='o', etag='',
                      properties={}, mime_type='application/octet-stream',
                      file_or_path=req.environ['wsgi.input'], policy=None,
-                     headers=ANY)
+                     headers=ANY, container_properties=ANY)
         self.assertEqual(201, resp.status_int)
         self.assertIn('Last-Modified', resp.headers)
         self.assertIn('Etag', resp.headers)