From dc7c51ed5dfc83a6e38762a0dd44cc0a94f009ac Mon Sep 17 00:00:00 2001 From: HuseinA Date: Wed, 27 May 2020 12:00:15 +0700 Subject: [PATCH 1/4] Add: upload_fileobj function --- obs/libs/bucket.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/obs/libs/bucket.py b/obs/libs/bucket.py index 6f50977..ec7fa85 100644 --- a/obs/libs/bucket.py +++ b/obs/libs/bucket.py @@ -125,13 +125,30 @@ def upload_object(**kwargs): bucket_name = kwargs.get("bucket_name") resource_upload = resource.Object(bucket_name, filename) if kwargs.get("content_type"): - resource_upload.upload_file( - Filename=local_path, ExtraArgs={"ContentType": kwargs.get("content_type")} + resource_upload.upload_fileobj( + local_path=local_path, ExtraArgs={"ContentType": kwargs.get("content_type")} ) else: resource_upload.upload_file(Filename=local_path) +def upload_bin_object(**kwargs): + """Upload an binary object into bucket.""" + fileobj = kwargs.get("fileobj") + filename = kwargs.get("object_name") + + resource = kwargs.get("resource") + bucket_name = kwargs.get("bucket_name") + resource_upload = resource.Object(bucket_name, filename) + + if kwargs.get("content_type"): + resource_upload.upload_fileobj( + Fileobj=fileobj, ExtraArgs={"ContentType": kwargs.get("content_type")} + ) + else: + resource_upload.upload_fileobj(Fileobj=fileobj) + + def copy_object(resource, src_bucket, src_object_name, dest_bucket, dest_object_name): """Copy an object into other bucket.""" From 5946c99bc0afaf741f772e1ca52bfbc1ba047476 Mon Sep 17 00:00:00 2001 From: HuseinA Date: Wed, 27 May 2020 12:00:24 +0700 Subject: [PATCH 2/4] Update: change how file treated in upload file --- obs/api/app/controllers/api/storage.py | 66 ++++++++++++++------------ 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/obs/api/app/controllers/api/storage.py b/obs/api/app/controllers/api/storage.py index ebdb27e..ef5b27e 100644 --- a/obs/api/app/controllers/api/storage.py +++ b/obs/api/app/controllers/api/storage.py @@ -4,7 +4,6 @@ import zipfile import tempfile import xmltodict -import mimetypes from obs.libs import bucket from obs.libs import gmt @@ -13,7 +12,7 @@ from requests_aws4auth import AWS4Auth from obs.api.app.helpers.rest import response from werkzeug.utils import secure_filename -from flask import request, send_file, current_app +from flask import request, current_app, send_from_directory from flask_restful import Resource, reqparse @@ -285,14 +284,26 @@ def get(self, bucket_name, name=""): name = archive(args["object_name"]) else: name = args["object_name"] - file = send_file(f"{tempdir}/{name}", as_attachment=True) + file = send_from_directory(tempdir, name, as_attachment=True) return file except Exception as e: - current_app.logger.error(f"{e}") + current_app.logger.error(f"{e}", exc_info=1) return response(500, f"{e}") class upload_object(Resource): + def get(self, bucket_name): + parser = reqparse.RequestParser() + parser.add_argument("access_key", type=str, required=True) + parser.add_argument("secret_key", type=str, required=True) + args = parser.parse_args() + secret_key = args["secret_key"].replace(" ", "+") + + objects = bucket.list_multipartupload( + get_resources(args["access_key"], secret_key), bucket_name + ) + return objects + def post(self, bucket_name): parser = reqparse.RequestParser() parser.add_argument("access_key", type=str, required=True) @@ -302,38 +313,33 @@ def post(self, bucket_name): args = parser.parse_args() secret_key = args["secret_key"].replace(" ", "+") - with tempfile.TemporaryDirectory() as tempdir: - file = request.files["files"] - filename = secure_filename(file.filename) - object_name = args["object_name"] if args["object_name"] else filename - filename = os.path.join(tempdir, filename) - file.save(filename) + file = request.files["files"] + filename = secure_filename(file.filename) + object_name = args["object_name"] if args["object_name"] else filename - try: - regex = r"[\"\{}^%`\]\[~<>|#]|[^\x00-\x7F]" - object_name = re.sub(regex, "", object_name) + try: + regex = r"[\"\{}^%`\]\[~<>|#]|[^\x00-\x7F]" + object_name = re.sub(regex, "", object_name) - result = bucket.upload_object( + result = bucket.upload_bin_object( + resource=get_resources(args["access_key"], secret_key), + bucket_name=bucket_name, + fileobj=file, + object_name=object_name, + content_type=file.content_type, + ) + if args["acl"]: + bucket.set_acl( resource=get_resources(args["access_key"], secret_key), bucket_name=bucket_name, - local_path=filename, object_name=object_name, - content_type=mimetypes.guess_type(filename)[0], - ) - if args["acl"]: - bucket.set_acl( - resource=get_resources(args["access_key"], secret_key), - bucket_name=bucket_name, - object_name=object_name, - acl_type="object", - acl=args["acl"], - ) - return response( - 201, f"Object {object_name} uploaded successfully.", result + acl_type="object", + acl=args["acl"], ) - except Exception as e: - current_app.logger.error(f"{e}") - return response(500, f"{e}") + return response(201, f"Object {object_name} uploaded successfully.", result) + except Exception as e: + current_app.logger.error(f"{e}", exc_info=1) + return response(500, f"{e}") class usage(Resource): From 1cb1cea07a3a846ce884528f6a0b66f2ce0f1172 Mon Sep 17 00:00:00 2001 From: HuseinA Date: Wed, 27 May 2020 12:00:52 +0700 Subject: [PATCH 3/4] Update: up version to v0.2.7 --- CHANGELOG.rst | 5 +++++ obs/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1ddd8e4..e23ac61 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,11 @@ Unreleased ========== +0.2.7 (2020-05-27) +================== +- add file_uploadobj in libs and use it in obs api +- add threads in gunicorn config + 0.2.6 (2020-05-12) ================== - fix api cannot access cloudian with secure url diff --git a/obs/__init__.py b/obs/__init__.py index fe404ae..6cd38b7 100644 --- a/obs/__init__.py +++ b/obs/__init__.py @@ -1 +1 @@ -__version__ = "0.2.5" +__version__ = "0.2.7" From 6bce0844d0c4f82d519d292d2523e925bfa04260 Mon Sep 17 00:00:00 2001 From: HuseinA Date: Wed, 27 May 2020 12:15:25 +0700 Subject: [PATCH 4/4] Fix: upload function --- obs/api/app/controllers/api/storage.py | 12 ------------ obs/libs/bucket.py | 4 ++-- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/obs/api/app/controllers/api/storage.py b/obs/api/app/controllers/api/storage.py index ef5b27e..70e5692 100644 --- a/obs/api/app/controllers/api/storage.py +++ b/obs/api/app/controllers/api/storage.py @@ -292,18 +292,6 @@ def get(self, bucket_name, name=""): class upload_object(Resource): - def get(self, bucket_name): - parser = reqparse.RequestParser() - parser.add_argument("access_key", type=str, required=True) - parser.add_argument("secret_key", type=str, required=True) - args = parser.parse_args() - secret_key = args["secret_key"].replace(" ", "+") - - objects = bucket.list_multipartupload( - get_resources(args["access_key"], secret_key), bucket_name - ) - return objects - def post(self, bucket_name): parser = reqparse.RequestParser() parser.add_argument("access_key", type=str, required=True) diff --git a/obs/libs/bucket.py b/obs/libs/bucket.py index ec7fa85..48e232d 100644 --- a/obs/libs/bucket.py +++ b/obs/libs/bucket.py @@ -125,8 +125,8 @@ def upload_object(**kwargs): bucket_name = kwargs.get("bucket_name") resource_upload = resource.Object(bucket_name, filename) if kwargs.get("content_type"): - resource_upload.upload_fileobj( - local_path=local_path, ExtraArgs={"ContentType": kwargs.get("content_type")} + resource_upload.upload_file( + Filename=local_path, ExtraArgs={"ContentType": kwargs.get("content_type")} ) else: resource_upload.upload_file(Filename=local_path)