Merge pull request #49 from HuseinA/master

Update: Multipart Upload/ Download

CHANGELOG.rst

@@ -1,6 +1,11 @@
 Unreleased
 ==========
 
+0.2.7 (2020-05-27)
+==================
+- add file_uploadobj in libs and use it in obs api
+- add threads in gunicorn config
+
 0.2.6 (2020-05-12)
 ==================
 - fix api cannot access cloudian with secure url

obs/__init__.py

@@ -1 +1 @@
-__version__ = "0.2.5"
+__version__ = "0.2.7"

obs/api/app/controllers/api/storage.py

@@ -4,7 +4,6 @@
 import zipfile
 import tempfile
 import xmltodict
-import mimetypes
 
 from obs.libs import bucket
 from obs.libs import gmt
@@ -13,7 +12,7 @@
 from requests_aws4auth import AWS4Auth
 from obs.api.app.helpers.rest import response
 from werkzeug.utils import secure_filename
-from flask import request, send_file, current_app
+from flask import request, current_app, send_from_directory
 from flask_restful import Resource, reqparse
 
 
@@ -285,10 +284,10 @@ def get(self, bucket_name, name=""):
                     name = archive(args["object_name"])
                 else:
                     name = args["object_name"]
-                file = send_file(f"{tempdir}/{name}", as_attachment=True)
+                file = send_from_directory(tempdir, name, as_attachment=True)
                 return file
             except Exception as e:
-                current_app.logger.error(f"{e}")
+                current_app.logger.error(f"{e}", exc_info=1)
                 return response(500, f"{e}")
 
 
@@ -302,38 +301,33 @@ def post(self, bucket_name):
         args = parser.parse_args()
         secret_key = args["secret_key"].replace(" ", "+")
 
-        with tempfile.TemporaryDirectory() as tempdir:
-            file = request.files["files"]
-            filename = secure_filename(file.filename)
-            object_name = args["object_name"] if args["object_name"] else filename
-            filename = os.path.join(tempdir, filename)
-            file.save(filename)
+        file = request.files["files"]
+        filename = secure_filename(file.filename)
+        object_name = args["object_name"] if args["object_name"] else filename
 
-            try:
-                regex = r"[\"\{}^%`\]\[~<>|#]|[^\x00-\x7F]"
-                object_name = re.sub(regex, "", object_name)
+        try:
+            regex = r"[\"\{}^%`\]\[~<>|#]|[^\x00-\x7F]"
+            object_name = re.sub(regex, "", object_name)
 
-                result = bucket.upload_object(
+            result = bucket.upload_bin_object(
+                resource=get_resources(args["access_key"], secret_key),
+                bucket_name=bucket_name,
+                fileobj=file,
+                object_name=object_name,
+                content_type=file.content_type,
+            )
+            if args["acl"]:
+                bucket.set_acl(
                     resource=get_resources(args["access_key"], secret_key),
                     bucket_name=bucket_name,
-                    local_path=filename,
                     object_name=object_name,
-                    content_type=mimetypes.guess_type(filename)[0],
+                    acl_type="object",
+                    acl=args["acl"],
                 )
-                if args["acl"]:
-                    bucket.set_acl(
-                        resource=get_resources(args["access_key"], secret_key),
-                        bucket_name=bucket_name,
-                        object_name=object_name,
-                        acl_type="object",
-                        acl=args["acl"],
-                    )
-                return response(
-                    201, f"Object {object_name} uploaded successfully.", result
-                )
-            except Exception as e:
-                current_app.logger.error(f"{e}")
-                return response(500, f"{e}")
+            return response(201, f"Object {object_name} uploaded successfully.", result)
+        except Exception as e:
+            current_app.logger.error(f"{e}", exc_info=1)
+            return response(500, f"{e}")
 
 
 class usage(Resource):

obs/libs/bucket.py

@@ -132,6 +132,23 @@ def upload_object(**kwargs):
         resource_upload.upload_file(Filename=local_path)
 
 
+def upload_bin_object(**kwargs):
+    """Upload an binary object into bucket."""
+    fileobj = kwargs.get("fileobj")
+    filename = kwargs.get("object_name")
+
+    resource = kwargs.get("resource")
+    bucket_name = kwargs.get("bucket_name")
+    resource_upload = resource.Object(bucket_name, filename)
+
+    if kwargs.get("content_type"):
+        resource_upload.upload_fileobj(
+            Fileobj=fileobj, ExtraArgs={"ContentType": kwargs.get("content_type")}
+        )
+    else:
+        resource_upload.upload_fileobj(Fileobj=fileobj)
+
+
 def copy_object(resource, src_bucket, src_object_name, dest_bucket, dest_object_name):
     """Copy an object into other bucket."""